From e974900314516818dcdd56ac92753c100e38c966 Mon Sep 17 00:00:00 2001 From: Ben Sherman Date: Sun, 19 Nov 2023 08:31:03 -0600 Subject: [PATCH 01/36] Refactor ast xform classes Signed-off-by: Ben Sherman --- .../src/main/groovy/nextflow/Nextflow.groovy | 10 +- .../nextflow/ast/BinaryExpressionXform.groovy | 119 ++ .../groovy/nextflow/ast/DslCodeVistor.groovy | 1248 ++++++++++++++++ .../groovy/nextflow/ast/NextflowDSL.groovy | 32 - .../nextflow/ast/NextflowDSLImpl.groovy | 1272 ----------------- .../nextflow/ast/NextflowXformImpl.groovy | 92 +- .../main/groovy/nextflow/ast/OpXform.groovy | 32 - ...pXformImpl.groovy => OperatorXform.groovy} | 137 +- .../nextflow/config/ConfigParser.groovy | 2 - .../config/ConfigTransformImpl.groovy | 2 + .../nextflow/processor/TaskConfig.groovy | 4 +- .../nextflow/processor/TaskProcessor.groovy | 4 +- .../groovy/nextflow/script/ChannelOut.groovy | 6 +- .../nextflow/script/ProcessConfig.groovy | 7 +- .../nextflow/script/ScriptParser.groovy | 4 - .../nextflow/script/ScriptTokens.groovy | 4 +- 16 files changed, 1454 insertions(+), 1521 deletions(-) create mode 100644 modules/nextflow/src/main/groovy/nextflow/ast/BinaryExpressionXform.groovy create mode 100644 modules/nextflow/src/main/groovy/nextflow/ast/DslCodeVistor.groovy delete mode 100644 modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSL.groovy delete mode 100644 modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSLImpl.groovy delete mode 100644 modules/nextflow/src/main/groovy/nextflow/ast/OpXform.groovy rename modules/nextflow/src/main/groovy/nextflow/ast/{OpXformImpl.groovy => OperatorXform.groovy} (90%) diff --git a/modules/nextflow/src/main/groovy/nextflow/Nextflow.groovy b/modules/nextflow/src/main/groovy/nextflow/Nextflow.groovy index 5976f229e3..97d5988b32 100644 --- a/modules/nextflow/src/main/groovy/nextflow/Nextflow.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/Nextflow.groovy @@ -24,8 +24,6 @@ import java.nio.file.NoSuchFileException import java.nio.file.Path import groovyx.gpars.dataflow.DataflowReadChannel -import nextflow.ast.OpXform -import nextflow.ast.OpXformImpl import nextflow.exception.StopSplitIterationException import nextflow.exception.WorkflowScriptErrorException import nextflow.extension.GroupKey @@ -381,11 +379,11 @@ class Nextflow { * Marker method to create a closure to be passed to {@link OperatorImpl#branch(DataflowReadChannel, groovy.lang.Closure)} * operator. * - * Despite apparently is doing nothing, this method is needed as marker to apply the {@link OpXform} AST + * Despite apparently is doing nothing, this method is needed as marker to apply the {@link OperatorXform} AST * transformation required to interpret the closure content as required for the branch evaluation. * * @see OperatorImpl#branch(DataflowReadChannel, Closure) - * @see OpXformImpl + * @see OperatorXform * * @param closure * @return @@ -396,11 +394,11 @@ class Nextflow { * Marker method to create a closure to be passed to {@link OperatorImpl#fork(DataflowReadChannel, Closure)} * operator. * - * Despite apparently is doing nothing, this method is needed as marker to apply the {@link OpXform} AST + * Despite apparently is doing nothing, this method is needed as marker to apply the {@link OperatorXform} AST * transformation required to interpret the closure content as required for the branch evaluation. * * @see OperatorImpl#multiMap(groovyx.gpars.dataflow.DataflowReadChannel, groovy.lang.Closure) (DataflowReadChannel, Closure) - * @see OpXformImpl + * @see OperatorXform * * @param closure * @return diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/BinaryExpressionXform.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/BinaryExpressionXform.groovy new file mode 100644 index 0000000000..ee5c282759 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/ast/BinaryExpressionXform.groovy @@ -0,0 +1,119 @@ +/* + * Copyright 2013-2023, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.ast + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import org.codehaus.groovy.ast.ClassCodeExpressionTransformer +import org.codehaus.groovy.ast.expr.BinaryExpression +import org.codehaus.groovy.ast.expr.ClosureExpression +import org.codehaus.groovy.ast.expr.Expression +import org.codehaus.groovy.ast.expr.MethodCallExpression +import org.codehaus.groovy.ast.expr.NotExpression +import org.codehaus.groovy.ast.tools.GeneralUtils +import org.codehaus.groovy.control.SourceUnit +/** + * Implements Nextflow Xform logic + + * See http://groovy-lang.org/metaprogramming.html#_classcodeexpressiontransformer + * + * @author Paolo Di Tommaso + */ +@Slf4j +@CompileStatic +class BinaryExpressionXform extends ClassCodeExpressionTransformer { + + private final SourceUnit unit + + BinaryExpressionXform(SourceUnit unit) { + this.unit = unit + } + + @Override + protected SourceUnit getSourceUnit() { unit } + + @Override + Expression transform(Expression expr) { + if (expr == null) + return null + + def newExpr = transformBinaryExpression(expr) + if( newExpr ) { + return newExpr + } + else if( expr instanceof ClosureExpression) { + visitClosureExpression(expr) + } + + return super.transform(expr) + } + + /** + * This method replaces the `==` with the invocation of + * {@link LangHelpers#compareEqual(java.lang.Object, java.lang.Object)} + * + * This is required to allow the comparisons of `Path` objects + * which by default are not supported because it implements the Comparator interface + * + * See + * {@link LangHelpers#compareEqual(java.lang.Object, java.lang.Object)} + * https://stackoverflow.com/questions/28355773/in-groovy-why-does-the-behaviour-of-change-for-interfaces-extending-compar#comment45123447_28387391 + * + */ + protected Expression transformBinaryExpression(Expression expr) { + + if( expr.class != BinaryExpression ) + return null + + def binary = expr as BinaryExpression + def left = binary.getLeftExpression() + def right = binary.getRightExpression() + + if( '=='.equals(binary.operation.text) ) + return call('compareEqual',left,right) + + if( '!='.equals(binary.operation.text) ) + return new NotExpression(call('compareEqual',left,right)) + + if( '<'.equals(binary.operation.text) ) + return call('compareLessThan', left,right) + + if( '<='.equals(binary.operation.text) ) + return call('compareLessThanEqual', left,right) + + if( '>'.equals(binary.operation.text) ) + return call('compareGreaterThan', left,right) + + if( '>='.equals(binary.operation.text) ) + return call('compareGreaterThanEqual', left,right) + + return null + } + + + private MethodCallExpression call(String method, Expression left, Expression right) { + + final a = transformBinaryExpression(left) ?: left + final b = transformBinaryExpression(right) ?: right + + GeneralUtils.callX( + GeneralUtils.classX(LangHelpers), + method, + GeneralUtils.args(a,b)) + } + +} diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/DslCodeVistor.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/DslCodeVistor.groovy new file mode 100644 index 0000000000..9dffe5ddbf --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/ast/DslCodeVistor.groovy @@ -0,0 +1,1248 @@ +/* + * Copyright 2013-2023, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.ast + +import static nextflow.Const.* +import static nextflow.ast.ASTHelpers.* +import static org.codehaus.groovy.ast.tools.GeneralUtils.* + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import nextflow.NF +import nextflow.script.BaseScript +import nextflow.script.BodyDef +import nextflow.script.IncludeDef +import nextflow.script.TaskClosure +import nextflow.script.TokenEnvCall +import nextflow.script.TokenFileCall +import nextflow.script.TokenPathCall +import nextflow.script.TokenStdinCall +import nextflow.script.TokenStdoutCall +import nextflow.script.TokenValCall +import nextflow.script.TokenValRef +import nextflow.script.TokenVar +import org.codehaus.groovy.ast.ASTNode +import org.codehaus.groovy.ast.ClassCodeVisitorSupport +import org.codehaus.groovy.ast.MethodNode +import org.codehaus.groovy.ast.Parameter +import org.codehaus.groovy.ast.VariableScope +import org.codehaus.groovy.ast.expr.ArgumentListExpression +import org.codehaus.groovy.ast.expr.BinaryExpression +import org.codehaus.groovy.ast.expr.CastExpression +import org.codehaus.groovy.ast.expr.ClosureExpression +import org.codehaus.groovy.ast.expr.ConstantExpression +import org.codehaus.groovy.ast.expr.Expression +import org.codehaus.groovy.ast.expr.GStringExpression +import org.codehaus.groovy.ast.expr.ListExpression +import org.codehaus.groovy.ast.expr.MapEntryExpression +import org.codehaus.groovy.ast.expr.MapExpression +import org.codehaus.groovy.ast.expr.MethodCallExpression +import org.codehaus.groovy.ast.expr.PropertyExpression +import org.codehaus.groovy.ast.expr.TupleExpression +import org.codehaus.groovy.ast.expr.UnaryMinusExpression +import org.codehaus.groovy.ast.expr.VariableExpression +import org.codehaus.groovy.ast.stmt.BlockStatement +import org.codehaus.groovy.ast.stmt.ExpressionStatement +import org.codehaus.groovy.ast.stmt.ReturnStatement +import org.codehaus.groovy.ast.stmt.Statement +import org.codehaus.groovy.control.SourceUnit +import org.codehaus.groovy.syntax.SyntaxException +import org.codehaus.groovy.syntax.Token +import org.codehaus.groovy.syntax.Types +/** + * Implements the syntax transformations for Nextflow DSL2. + * + * @author Paolo Di Tommaso + */ +@Slf4j +@CompileStatic +class DslCodeVisitor extends ClassCodeVisitorSupport { + + final static private String WORKFLOW_TAKE = 'take' + final static private String WORKFLOW_EMIT = 'emit' + final static private String WORKFLOW_MAIN = 'main' + final static private List SCOPES = [WORKFLOW_TAKE, WORKFLOW_EMIT, WORKFLOW_MAIN] + + final static public String PROCESS_WHEN = 'when' + final static public String PROCESS_STUB = 'stub' + + static public String OUT_PREFIX = '$out' + + static private Set RESERVED_NAMES + + static { + // method names implicitly defined by the groovy script SHELL + RESERVED_NAMES = ['main','run','runScript'] as Set + // existing method cannot be used for custom script definition + for( def method : BaseScript.getMethods() ) { + RESERVED_NAMES.add(method.name) + } + + } + + final private SourceUnit unit + + private String currentTaskName + + private String currentLabel + + private String bodyLabel + + private Set processNames = [] + + private Set workflowNames = [] + + private Set functionNames = [] + + private int anonymousWorkflow + + @Override + protected SourceUnit getSourceUnit() { unit } + + DslCodeVisitor(SourceUnit unit) { + this.unit = unit + } + + @Override + void visitMethod(MethodNode node) { + if( node.public && !node.static && !node.synthetic && !node.metaDataMap?.'org.codehaus.groovy.ast.MethodNode.isScriptBody') { + if( !isIllegalName(node.name, node)) + functionNames.add(node.name) + } + super.visitMethod(node) + } + + @Override + void visitMethodCallExpression(MethodCallExpression methodCall) { + // pre-condition to be verified to apply the transformation + final preCondition = methodCall.objectExpression?.getText() == 'this' + final methodName = methodCall.getMethodAsString() + + /* + * intercept the *process* method in order to transform the script closure + */ + if( methodName == 'process' && preCondition ) { + + // clear block label + bodyLabel = null + currentLabel = null + currentTaskName = methodName + try { + convertProcessDef(methodCall,sourceUnit) + super.visitMethodCallExpression(methodCall) + } + finally { + currentTaskName = null + } + } + else if( methodName == 'workflow' && preCondition ) { + convertWorkflowDef(methodCall,sourceUnit) + super.visitMethodCallExpression(methodCall) + } + + // just apply the default behavior + else { + super.visitMethodCallExpression(methodCall) + } + + } + + @Override + void visitExpressionStatement(ExpressionStatement stm) { + if( stm.text.startsWith('this.include(') && stm.getExpression() instanceof MethodCallExpression ) { + final methodCall = (MethodCallExpression)stm.getExpression() + convertIncludeDef(methodCall) + // this is necessary to invoke the `load` method on the include definition + final loadCall = new MethodCallExpression(methodCall, 'load0', new ArgumentListExpression(new VariableExpression('params'))) + stm.setExpression(loadCall) + } + super.visitExpressionStatement(stm) + } + + protected void convertIncludeDef(MethodCallExpression call) { + if( call.methodAsString=='include' && call.arguments instanceof ArgumentListExpression ) { + final allArgs = (ArgumentListExpression)call.arguments + if( allArgs.size() != 1 ) { + syntaxError(call, "Not a valid include definition -- it must specify the module path") + return + } + + final arg = allArgs[0] + final newArgs = new ArgumentListExpression() + if( arg instanceof ConstantExpression ) { + newArgs.addExpression( createX(IncludeDef, arg) ) + } + else if( arg instanceof VariableExpression ) { + // the name of the component i.e. process, workflow, etc to import + final component = arg.getName() + // wrap the name in a `TokenVar` type + final token = createX(TokenVar, new ConstantExpression(component)) + // create a new `IncludeDef` object + newArgs.addExpression(createX(IncludeDef, token)) + } + else if( arg instanceof CastExpression && arg.getExpression() instanceof VariableExpression) { + def cast = (CastExpression)arg + // the name of the component i.e. process, workflow, etc to import + final component = (cast.expression as VariableExpression).getName() + // wrap the name in a `TokenVar` type + final token = createX(TokenVar, new ConstantExpression(component)) + // the alias to give it + final alias = constX(cast.type.name) + newArgs.addExpression( createX(IncludeDef, token, alias) ) + } + else if( arg instanceof ClosureExpression ) { + // multiple modules inclusion + final block = (BlockStatement)arg.getCode() + final modulesList = new ListExpression() + for( Statement stm : block.statements ) { + if( stm instanceof ExpressionStatement ) { + CastExpression castX + VariableExpression varX + Expression moduleX + if( (varX=isVariableX(stm.expression)) ) { + def name = constX(varX.name) + moduleX = createX(IncludeDef.Module, name) + } + else if( (castX=isCastX(stm.expression)) && (varX=isVariableX(castX.expression)) ) { + def name = constX(varX.name) + final alias = constX(castX.type.name) + moduleX = createX(IncludeDef.Module, name, alias) + } + else { + syntaxError(call, "Not a valid include module name") + return + } + modulesList.addExpression(moduleX) + } + else { + syntaxError(call, "Not a valid include module name") + return + } + + } + newArgs.addExpression( createX(IncludeDef, modulesList) ) + } + else { + syntaxError(call, "Not a valid include definition -- it must specify the module path as a string") + return + } + call.setArguments(newArgs) + } + else if( call.objectExpression instanceof MethodCallExpression ) { + convertIncludeDef((MethodCallExpression)call.objectExpression) + } + } + + /* + * this method transforms the DSL definition + * + * workflow foo { + * code + * } + * + * into a method invocation as + * + * workflow('foo', { -> code }) + * + */ + protected void convertWorkflowDef(MethodCallExpression methodCall, SourceUnit unit) { + log.trace "Convert 'workflow' ${methodCall.arguments}" + + assert methodCall.arguments instanceof ArgumentListExpression + def args = (ArgumentListExpression)methodCall.arguments + def len = args.size() + + // anonymous workflow definition + if( len == 1 && args[0] instanceof ClosureExpression ) { + if( anonymousWorkflow++ > 0 ) { + unit.addError( new SyntaxException("Duplicate entry workflow definition", methodCall.lineNumber, methodCall.columnNumber+8)) + return + } + + def newArgs = new ArgumentListExpression() + def body = (ClosureExpression)args[0] + newArgs.addExpression( makeWorkflowDefWrapper(body,true) ) + methodCall.setArguments( newArgs ) + return + } + + // extract the first argument which has to be a method-call expression + // the name of this method represent the *workflow* name + if( len != 1 || !args[0].class.isAssignableFrom(MethodCallExpression) ) { + log.debug "Missing name in workflow definition at line: ${methodCall.lineNumber}" + unit.addError( new SyntaxException("Workflow definition syntax error -- A string identifier must be provided after the `workflow` keyword", methodCall.lineNumber, methodCall.columnNumber+8)) + return + } + + final nested = args[0] as MethodCallExpression + final name = nested.getMethodAsString() + // check the process name is not defined yet + if( isIllegalName(name, methodCall) ) { + return + } + workflowNames.add(name) + + // the nested method arguments are the arguments to be passed + // to the process definition, plus adding the process *name* + // as an extra item in the arguments list + args = (ArgumentListExpression)nested.getArguments() + len = args.size() + log.trace "Workflow name: $name with args: $args" + + // make sure to add the 'name' after the map item + // (which represent the named parameter attributes) + def newArgs = new ArgumentListExpression() + + // add the workflow body def + if( len != 1 || !(args[0] instanceof ClosureExpression)) { + syntaxError(methodCall, "Invalid workflow definition") + return + } + + final body = (ClosureExpression)args[0] + newArgs.addExpression( constX(name) ) + newArgs.addExpression( makeWorkflowDefWrapper(body,false) ) + + // set the new list as the new arguments + methodCall.setArguments( newArgs ) + } + + + protected Statement normWorkflowParam(ExpressionStatement stat, String type, Set uniqueNames, List body) { + MethodCallExpression callx + VariableExpression varx + + if( (callx=isMethodCallX(stat.expression)) && isThisX(callx.objectExpression) ) { + final name = "_${type}_${callx.methodAsString}" + return stmt( callThisX(name, callx.arguments) ) + } + + if( (varx=isVariableX(stat.expression)) ) { + final name = "_${type}_${varx.name}" + return stmt( callThisX(name) ) + } + + if( type == WORKFLOW_EMIT ) { + return createAssignX(stat, body, type, uniqueNames) + } + + syntaxError(stat, "Workflow malformed parameter definition") + return stat + } + + protected Statement createAssignX(ExpressionStatement stat, List body, String type, Set uniqueNames) { + BinaryExpression binx + MethodCallExpression callx + Expression args=null + + if( (binx=isAssignX(stat.expression)) ) { + // keep the statement in body to allow it to be evaluated + body.add(stat) + // and create method call expr to capture the var name in the emission + final left = (VariableExpression)binx.leftExpression + final name = "_${type}_${left.name}" + return stmt( callThisX(name) ) + } + + if( (callx=isMethodCallX(stat.expression)) && callx.objectExpression.text!='this' && hasTo(callx)) { + // keep the args + args = callx.arguments + // replace the method call expression with a property + stat.expression = new PropertyExpression(callx.objectExpression, callx.method) + // then, fallback to default case + } + + // wrap the expression into a assignment expression + final var = getNextName(uniqueNames) + final left = new VariableExpression(var) + final right = stat.expression + final token = new Token(Types.ASSIGN, '=', -1, -1) + final assign = new BinaryExpression(left, token, right) + body.add(stmt(assign)) + + // the call method statement for the emit declaration + final name="_${type}_${var}" + callx = args ? callThisX(name, args) : callThisX(name) + return stmt(callx) + } + + protected boolean hasTo(MethodCallExpression callX) { + def tupleX = isTupleX(callX.arguments) + if( !tupleX ) return false + if( !tupleX.expressions ) return false + def mapX = isMapX(tupleX.expressions[0]) + if( !mapX ) return false + def entry = mapX.getMapEntryExpressions().find { isConstX(it.keyExpression).text=='to' } + return entry != null + } + + protected String getNextName(Set allNames) { + String result + while( true ) { + result = OUT_PREFIX + allNames.size() + if( allNames.add(result) ) + break + } + return result + } + + protected Expression makeWorkflowDefWrapper( ClosureExpression closure, boolean anonymous ) { + + final codeBlock = (BlockStatement) closure.code + final codeStms = codeBlock.statements + final scope = codeBlock.variableScope + + final visited = new HashMap(5); + final emitNames = new LinkedHashSet(codeStms.size()) + final wrap = new ArrayList(codeStms.size()) + final body = new ArrayList(codeStms.size()) + final source = new StringBuilder() + String context = null + String previous = null + for( Statement stm : codeStms ) { + previous = context + context = stm.statementLabel ?: context + // check for changing context + if( context && context != previous ) { + if( visited[context] && visited[previous] ) { + syntaxError(stm, "Unexpected workflow `${context}` context here") + break + } + } + visited[context] = true + + switch (context) { + case WORKFLOW_TAKE: + case WORKFLOW_EMIT: + if( !(stm instanceof ExpressionStatement) ) { + syntaxError(stm, "Workflow malformed parameter definition") + break + } + wrap.add(normWorkflowParam(stm as ExpressionStatement, context, emitNames, body)) + break + + case WORKFLOW_MAIN: + body.add(stm) + break + + default: + if( context ) { + def opts = SCOPES.closest(context) + def msg = "Unknown execution scope '$context:'" + if( opts ) msg += " -- Did you mean ${opts.collect{"'$it'"}.join(', ')}" + syntaxError(stm, msg) + } + body.add(stm) + } + } + // read the closure source + readSource(closure, source, unit, true) + + final bodyClosure = closureX(null, block(scope, body)) + final invokeBody = makeScriptWrapper(bodyClosure, source.toString(), 'workflow', unit) + wrap.add( stmt(invokeBody) ) + + closureX(null, block(scope, wrap)) + } + + protected void syntaxError(ASTNode node, String message) { + int line = node.lineNumber + int coln = node.columnNumber + unit.addError( new SyntaxException(message,line,coln)) + } + + /** + * Transform a DSL `process` definition into a proper method invocation + * + * @param methodCall + * @param unit + */ + protected void convertProcessBlock( MethodCallExpression methodCall, SourceUnit unit ) { + log.trace "Apply task closure transformation to method call: $methodCall" + + final args = methodCall.arguments as ArgumentListExpression + final lastArg = args.expressions.size()>0 ? args.getExpression(args.expressions.size()-1) : null + final isClosure = lastArg instanceof ClosureExpression + + if( isClosure ) { + // the block holding all the statements defined in the process (closure) definition + final block = (lastArg as ClosureExpression).code as BlockStatement + + /* + * iterate over the list of statements to: + * - converts the method after the 'input:' label as input parameters + * - converts the method after the 'output:' label as output parameters + * - collect all the statement after the 'exec:' label + */ + def source = new StringBuilder() + List execStatements = [] + + List whenStatements = [] + def whenSource = new StringBuilder() + + List stubStatements = [] + def stubSource = new StringBuilder() + + + def iterator = block.getStatements().iterator() + while( iterator.hasNext() ) { + + // get next statement + Statement stm = iterator.next() + + // keep track of current block label + currentLabel = stm.statementLabel ?: currentLabel + + switch(currentLabel) { + case 'input': + if( stm instanceof ExpressionStatement ) { + fixLazyGString( stm ) + fixStdinStdout( stm ) + convertInputMethod( stm.getExpression() ) + } + break + + case 'output': + if( stm instanceof ExpressionStatement ) { + fixLazyGString( stm ) + fixStdinStdout( stm ) + convertOutputMethod( stm.getExpression() ) + } + break + + case 'exec': + bodyLabel = currentLabel + iterator.remove() + execStatements << stm + readSource(stm,source,unit) + break + + case 'script': + case 'shell': + bodyLabel = currentLabel + iterator.remove() + execStatements << stm + readSource(stm,source,unit) + break + + case PROCESS_STUB: + iterator.remove() + stubStatements << stm + readSource(stm,stubSource,unit) + break + + // capture the statements in a when guard and remove from the current block + case PROCESS_WHEN: + if( iterator.hasNext() ) { + iterator.remove() + whenStatements << stm + readSource(stm,whenSource,unit) + break + } + // when entering in this branch means that this is the last statement, + // which is supposed to be the task command + // hence if no previous `when` statement has been processed, a syntax error is returned + else if( !whenStatements ) { + int line = methodCall.lineNumber + int coln = methodCall.columnNumber + unit.addError(new SyntaxException("Invalid process definition -- Empty `when` or missing `script` statement", line, coln)) + return + } + else + break + + default: + if(currentLabel) { + def line = stm.getLineNumber() + def coln = stm.getColumnNumber() + unit.addError(new SyntaxException("Invalid process definition -- Unknown keyword `$currentLabel`",line,coln)) + return + } + + fixLazyGString(stm) + fixDirectiveWithNegativeValue(stm) // Fixes #180 + } + } + + /* + * add the `when` block if found + */ + if( whenStatements ) { + addWhenGuardCall(whenStatements, whenSource, block) + } + + /* + * add try `stub` block if found + */ + if( stubStatements ) { + final newBLock = addStubCall(stubStatements, stubSource, block) + newBLock.visit(new TaskCmdXformVisitor(unit)) + } + + /* + * wrap all the statements after the 'exec:' label by a new closure containing them (in a new block) + */ + final len = block.statements.size() + boolean done = false + if( execStatements ) { + // create a new Closure + def execBlock = new BlockStatement(execStatements, new VariableScope(block.variableScope)) + def execClosure = new ClosureExpression( Parameter.EMPTY_ARRAY, execBlock ) + + // append the new block to the + // set the 'script' flag parameter + def wrap = makeScriptWrapper(execClosure, source, bodyLabel, unit) + block.addStatement( new ExpressionStatement(wrap) ) + if( bodyLabel == 'script' ) + block.visit(new TaskCmdXformVisitor(unit)) + done = true + + } + // when only the `stub` block is defined add an empty command + else if ( !bodyLabel && stubStatements ) { + final cmd = 'true' + final list = new ArrayList(1); + list.add( new ExpressionStatement(constX(cmd)) ) + final dummyBlock = new BlockStatement( list, new VariableScope(block.variableScope)) + final dummyClosure = new ClosureExpression( Parameter.EMPTY_ARRAY, dummyBlock ) + + // append the new block to the + // set the 'script' flag parameter + final wrap = makeScriptWrapper(dummyClosure, cmd, 'script', unit) + block.addStatement( new ExpressionStatement(wrap) ) + done = true + } + + /* + * when the last statement is a string script, the 'script:' label can be omitted + */ + else if( len ) { + def stm = block.getStatements().get(len-1) + readSource(stm,source,unit) + + if ( stm instanceof ReturnStatement ){ + done = wrapExpressionWithClosure(block, stm.getExpression(), len, source, unit) + } + + else if ( stm instanceof ExpressionStatement ) { + done = wrapExpressionWithClosure(block, stm.getExpression(), len, source, unit) + } + + // apply command variables escape + stm.visit(new TaskCmdXformVisitor(unit)) + } + + if (!done) { + log.trace "Invalid 'process' definition -- Process must terminate with string expression" + int line = methodCall.lineNumber + int coln = methodCall.columnNumber + unit.addError( new SyntaxException("Invalid process definition -- Make sure the process ends with a script wrapped by quote characters",line,coln)) + } + } + } + + /** + * Converts a `when` block into a when method call expression. The when code is converted into a + * closure expression and set a `when` directive in the process configuration properties. + * + * See {@link nextflow.script.ProcessConfig#configProperties} + * See {@link nextflow.processor.TaskConfig#getGuard(java.lang.String)} + */ + protected BlockStatement addWhenGuardCall( List statements, StringBuilder source, BlockStatement parent ) { + createBlock0(PROCESS_WHEN, statements, source, parent) + } + + protected BlockStatement addStubCall(List statements, StringBuilder source, BlockStatement parent ) { + createBlock0(PROCESS_STUB, statements, source, parent) + } + + protected BlockStatement createBlock0( String blockName, List statements, StringBuilder source, BlockStatement parent ) { + // wrap the code block into a closure expression + def block = new BlockStatement(statements, new VariableScope(parent.variableScope)) + def closure = new ClosureExpression( Parameter.EMPTY_ARRAY, block ) + + // the closure expression is wrapped itself into a TaskClosure object + // in order to capture the closure source other than the closure code + List newArgs = [] + newArgs << closure + newArgs << new ConstantExpression(source.toString()) + def whenObj = createX( TaskClosure, newArgs ) + + // creates a method call expression for the method `when` + def method = new MethodCallExpression(VariableExpression.THIS_EXPRESSION, blockName, whenObj) + parent.getStatements().add(0, new ExpressionStatement(method)) + + return block + } + + /** + * Wrap the user provided piece of code, either a script or a closure with a {@code BodyDef} object + * + * @param closure + * @param source + * @param scriptOrNative + * @param unit + * @return a {@code BodyDef} object + */ + private Expression makeScriptWrapper( ClosureExpression closure, CharSequence source, String section, SourceUnit unit ) { + + final List newArgs = [] + newArgs << (closure) + newArgs << ( new ConstantExpression(source.toString()) ) + newArgs << ( new ConstantExpression(section) ) + + // collect all variable tokens and pass them as single list argument + final variables = fetchVariables(closure,unit) + final listArg = new ArrayList(variables.size()) + for( TokenValRef var: variables ) { + def pName = new ConstantExpression(var.name) + def pLine = new ConstantExpression(var.lineNum) + def pCol = new ConstantExpression(var.colNum) + listArg << createX( TokenValRef, pName, pLine, pCol ) + } + newArgs << ( new ListExpression(listArg) ) + + // invokes the BodyDef constructor + createX( BodyDef, newArgs ) + } + + /** + * Read the user provided script source string + * + * @param node + * @param buffer + * @param unit + */ + private void readSource( ASTNode node, StringBuilder buffer, SourceUnit unit, stripBrackets=false ) { + final colx = node.getColumnNumber() + final colz = node.getLastColumnNumber() + final first = node.getLineNumber() + final last = node.getLastLineNumber() + for( int i=first; i<=last; i++ ) { + def line = unit.source.getLine(i, null) + if( i==last ) { + line = line.substring(0,colz-1) + if( stripBrackets ) { + line = line.replaceFirst(/}.*$/,'') + if( !line.trim() ) continue + } + } + if( i==first ) { + line = line.substring(colx-1) + if( stripBrackets ) { + line = line.replaceFirst(/^.*\{/,'').trim() + if( !line.trim() ) continue + } + } + buffer.append(line) .append('\n') + } + } + + protected void fixLazyGString( Statement stm ) { + if( stm instanceof ExpressionStatement && stm.getExpression() instanceof MethodCallExpression ) { + new GStringToLazyVisitor(unit).visitExpressionStatement(stm) + } + } + + protected void fixDirectiveWithNegativeValue( Statement stm ) { + if( stm instanceof ExpressionStatement && stm.getExpression() instanceof BinaryExpression ) { + def binary = (BinaryExpression)stm.getExpression() + if(!(binary.leftExpression instanceof VariableExpression)) + return + if( binary.operation.type != Types.MINUS ) + return + + // -- transform the binary expression into a method call expression + // where the left expression represents the method name to invoke + def methodName = ((VariableExpression)binary.leftExpression).name + + // -- wrap the value into a minus operator + def value = (Expression)new UnaryMinusExpression( binary.rightExpression ) + def args = new ArgumentListExpression( [value] ) + + // -- create the method call expression and replace it to the binary expression + def call = new MethodCallExpression(new VariableExpression('this'), methodName, args) + stm.setExpression(call) + + } + } + + protected void fixStdinStdout( ExpressionStatement stm ) { + + // transform the following syntax: + // `stdin from x` --> stdin() from (x) + // `stdout into x` --> `stdout() into (x)` + VariableExpression varX + if( stm.expression instanceof PropertyExpression ) { + def expr = (PropertyExpression)stm.expression + def obj = expr.objectExpression + def prop = expr.property as ConstantExpression + def target = new VariableExpression(prop.text) + + if( obj instanceof MethodCallExpression ) { + def methodCall = obj as MethodCallExpression + if( 'stdout' == methodCall.getMethodAsString() ) { + def stdout = new MethodCallExpression( new VariableExpression('this'), 'stdout', new ArgumentListExpression() ) + def into = new MethodCallExpression(stdout, 'into', new ArgumentListExpression(target)) + // remove replace the old one with the new one + stm.setExpression( into ) + } + else if( 'stdin' == methodCall.getMethodAsString() ) { + def stdin = new MethodCallExpression( new VariableExpression('this'), 'stdin', new ArgumentListExpression() ) + def from = new MethodCallExpression(stdin, 'from', new ArgumentListExpression(target)) + // remove replace the old one with the new one + stm.setExpression( from ) + } + } + } + // transform the following syntax: + // `stdout into (x,y,..)` --> `stdout() into (x,y,..)` + else if( stm.expression instanceof MethodCallExpression ) { + def methodCall = (MethodCallExpression)stm.expression + if( 'stdout' == methodCall.getMethodAsString() ) { + def args = methodCall.getArguments() + if( args instanceof ArgumentListExpression && args.getExpressions() && args.getExpression(0) instanceof MethodCallExpression ) { + def methodCall2 = (MethodCallExpression)args.getExpression(0) + def args2 = methodCall2.getArguments() + if( args2 instanceof ArgumentListExpression && methodCall2.methodAsString == 'into') { + def vars = args2.getExpressions() + def stdout = new MethodCallExpression( new VariableExpression('this'), 'stdout', new ArgumentListExpression() ) + def into = new MethodCallExpression(stdout, 'into', new ArgumentListExpression(vars)) + // remove replace the old one with the new one + stm.setExpression( into ) + } + } + } + } + else if( (varX=isVariableX(stm.expression)) && (varX.name=='stdin' || varX.name=='stdout') && NF.isDsl2() ) { + final name = varX.name=='stdin' ? '_in_stdin' : '_out_stdout' + final call = new MethodCallExpression( new VariableExpression('this'), name, new ArgumentListExpression() ) + // remove replace the old one with the new one + stm.setExpression(call) + } + } + + /* + * handle *input* parameters + */ + protected void convertInputMethod( Expression expression ) { + log.trace "convert > input expression: $expression" + + if( expression instanceof MethodCallExpression ) { + + def methodCall = expression as MethodCallExpression + def methodName = methodCall.getMethodAsString() + def nested = methodCall.objectExpression instanceof MethodCallExpression + log.trace "convert > input method: $methodName" + + if( methodName in ['val','env','file','each','set','stdin','path','tuple'] ) { + //this methods require a special prefix + if( !nested ) + methodCall.setMethod( new ConstantExpression('_in_' + methodName) ) + + fixMethodCall(methodCall) + } + + /* + * Handles a GString a file name, like this: + * + * input: + * file x name "$var_name" from q + * + */ + else if( methodName == 'name' && isWithinMethod(expression, 'file') ) { + varToConstX(methodCall.getArguments()) + } + + // invoke on the next method call + if( expression.objectExpression instanceof MethodCallExpression ) { + convertInputMethod(methodCall.objectExpression) + } + } + + else if( expression instanceof PropertyExpression ) { + // invoke on the next method call + if( expression.objectExpression instanceof MethodCallExpression ) { + convertInputMethod(expression.objectExpression) + } + } + + } + + protected boolean isWithinMethod(MethodCallExpression method, String name) { + if( method.objectExpression instanceof MethodCallExpression ) { + return isWithinMethod(method.objectExpression as MethodCallExpression, name) + } + + return method.getMethodAsString() == name + } + + /** + * Transform a map entry `emit: something` into `emit: 'something' + * (ie. as a constant) in a map expression passed as argument to + * a method call. This allow the syntax + * + * output: + * path 'foo', emit: bar + * + * @param call + */ + protected void fixOutEmitOption(MethodCallExpression call) { + List args = isTupleX(call.arguments)?.expressions + if( !args ) return + if( args.size()<2 && (args.size()!=1 || call.methodAsString!='_out_stdout')) return + MapExpression map = isMapX(args[0]) + if( !map ) return + for( int i=0; i output expression: $expression" + + if( !(expression instanceof MethodCallExpression) ) { + return + } + + def methodCall = expression as MethodCallExpression + def methodName = methodCall.getMethodAsString() + def nested = methodCall.objectExpression instanceof MethodCallExpression + log.trace "convert > output method: $methodName" + + if( methodName in ['val','env','file','set','stdout','path','tuple'] && !nested ) { + // prefix the method name with the string '_out_' + methodCall.setMethod( new ConstantExpression('_out_' + methodName) ) + fixMethodCall(methodCall) + fixOutEmitOption(methodCall) + } + + else if( methodName in ['into','mode'] ) { + fixMethodCall(methodCall) + } + + // continue to traverse + if( methodCall.objectExpression instanceof MethodCallExpression ) { + convertOutputMethod(methodCall.objectExpression) + } + + } + + private boolean withinTupleMethod + + private boolean withinEachMethod + + /** + * This method converts the a method call argument from a Variable to a Constant value + * so that it is possible to reference variable that not yet exist + * + * @param methodCall The method object for which it is required to change args definition + * @param flagVariable Whenever append a flag specified if the variable replacement has been applied + * @param index The index of the argument to modify + * @return + */ + protected void fixMethodCall( MethodCallExpression methodCall ) { + final name = methodCall.methodAsString + + withinTupleMethod = name == '_in_set' || name == '_out_set' || name == '_in_tuple' || name == '_out_tuple' + withinEachMethod = name == '_in_each' + + try { + if( isOutputWithPropertyExpression(methodCall) ) { + // transform an output value declaration such + // output: val( obj.foo ) + // to + // output: val({ obj.foo }) + wrapPropertyToClosure((ArgumentListExpression)methodCall.getArguments()) + } + else + varToConstX(methodCall.getArguments()) + + } finally { + withinTupleMethod = false + withinEachMethod = false + } + } + + static final private List OUT_PROPERTY_VALID_TYPES = ['_out_val', '_out_env', '_out_file', '_out_path'] + + protected boolean isOutputWithPropertyExpression(MethodCallExpression methodCall) { + if( methodCall.methodAsString !in OUT_PROPERTY_VALID_TYPES ) + return false + if( methodCall.getArguments() instanceof ArgumentListExpression ) { + def args = (ArgumentListExpression)methodCall.getArguments() + if( args.size()==0 || args.size()>2 ) + return false + + return args.last() instanceof PropertyExpression + } + + return false + } + + protected void wrapPropertyToClosure(ArgumentListExpression expr) { + final args = expr as ArgumentListExpression + final property = (PropertyExpression) args.last() + final closure = wrapPropertyToClosure(property) + args.getExpressions().set(args.size()-1, closure) + } + + protected ClosureExpression wrapPropertyToClosure(PropertyExpression property) { + def block = new BlockStatement() + block.addStatement( new ExpressionStatement(property) ) + + def closure = new ClosureExpression( Parameter.EMPTY_ARRAY, block ) + closure.variableScope = new VariableScope(block.variableScope) + + return closure + } + + + protected Expression varToStrX( Expression expr ) { + if( expr instanceof VariableExpression ) { + def name = ((VariableExpression) expr).getName() + return createX( TokenVar, new ConstantExpression(name) ) + } + else if( expr instanceof PropertyExpression ) { + // transform an output declaration such + // output: tuple val( obj.foo ) + // to + // output: tuple val({ obj.foo }) + return wrapPropertyToClosure(expr) + } + + if( expr instanceof TupleExpression ) { + def i = 0 + def list = expr.getExpressions() + for( Expression item : list ) { + list[i++] = varToStrX(item) + } + + return expr + } + + return expr + } + + protected Expression varToConstX( Expression expr ) { + + if( expr instanceof VariableExpression ) { + // when it is a variable expression, replace it with a constant representing + // the variable name + def name = ((VariableExpression) expr).getName() + + /* + * the 'stdin' is used as placeholder for the standard input in the tuple definition. For example: + * + * input: + * tuple( stdin, .. ) from q + */ + if( name == 'stdin' && withinTupleMethod ) + return createX( TokenStdinCall ) + + /* + * input: + * tuple( stdout, .. ) + */ + else if ( name == 'stdout' && withinTupleMethod ) + return createX( TokenStdoutCall ) + + else + return createX( TokenVar, new ConstantExpression(name) ) + } + + if( expr instanceof MethodCallExpression ) { + def methodCall = expr as MethodCallExpression + + /* + * replace 'file' method call in the tuple definition, for example: + * + * input: + * tuple( file(fasta:'*.fa'), .. ) from q + */ + if( methodCall.methodAsString == 'file' && (withinTupleMethod || withinEachMethod) ) { + def args = (TupleExpression) varToConstX(methodCall.arguments) + return createX( TokenFileCall, args ) + } + else if( methodCall.methodAsString == 'path' && (withinTupleMethod || withinEachMethod) ) { + def args = (TupleExpression) varToConstX(methodCall.arguments) + return createX( TokenPathCall, args ) + } + + /* + * input: + * tuple( env(VAR_NAME) ) from q + */ + if( methodCall.methodAsString == 'env' && withinTupleMethod ) { + def args = (TupleExpression) varToStrX(methodCall.arguments) + return createX( TokenEnvCall, args ) + } + + /* + * input: + * tuple val(x), .. from q + */ + if( methodCall.methodAsString == 'val' && withinTupleMethod ) { + def args = (TupleExpression) varToStrX(methodCall.arguments) + return createX( TokenValCall, args ) + } + + } + + // -- TupleExpression or ArgumentListExpression + if( expr instanceof TupleExpression ) { + def i = 0 + def list = expr.getExpressions() + for( Expression item : list ) { + list[i++] = varToConstX(item) + } + return expr + } + + return expr + } + + /** + * Wrap a generic expression with in a closure expression + * + * @param block The block to which the resulting closure has to be appended + * @param expr The expression to the wrapped in a closure + * @param len + * @return A tuple in which: + *
  • 1st item: {@code true} if successful or {@code false} otherwise + *
  • 2nd item: on error condition the line containing the error in the source script, zero otherwise + *
  • 3rd item: on error condition the column containing the error in the source script, zero otherwise + * + */ + protected boolean wrapExpressionWithClosure( BlockStatement block, Expression expr, int len, CharSequence source, SourceUnit unit ) { + if( expr instanceof GStringExpression || expr instanceof ConstantExpression ) { + // remove the last expression + block.statements.remove(len-1) + + // and replace it by a wrapping closure + def closureExp = new ClosureExpression( Parameter.EMPTY_ARRAY, new ExpressionStatement(expr) ) + closureExp.variableScope = new VariableScope(block.variableScope) + + // append to the list of statement + //def wrap = newObj(BodyDef, closureExp, new ConstantExpression(source.toString()), ConstantExpression.TRUE) + def wrap = makeScriptWrapper(closureExp, source, 'script', unit ) + block.statements.add( new ExpressionStatement(wrap) ) + + return true + } + else if( expr instanceof ClosureExpression ) { + // do not touch it + return true + } + else { + log.trace "Invalid process result expression: ${expr} -- Only constant or string expression can be used" + } + + return false + } + + protected boolean isIllegalName(String name, ASTNode node) { + if( name in RESERVED_NAMES ) { + unit.addError( new SyntaxException("Identifier `$name` is reserved for internal use", node.lineNumber, node.columnNumber+8) ) + return true + } + if( name in workflowNames || name in processNames ) { + unit.addError( new SyntaxException("Identifier `$name` is already used by another definition", node.lineNumber, node.columnNumber+8) ) + return true + } + if( name.contains(SCOPE_SEP) ) { + def offset = 8+2+ name.indexOf(SCOPE_SEP) + unit.addError( new SyntaxException("Process and workflow names cannot contain colon character", node.lineNumber, node.columnNumber+offset) ) + return true + } + return false + } + + /** + * This method handle the process definition, so that it transform the user entered syntax + * process myName ( named: args, .. ) { code .. } + * + * into + * process ( [named:args,..], String myName ) { } + * + * @param methodCall + * @param unit + */ + protected void convertProcessDef( MethodCallExpression methodCall, SourceUnit unit ) { + log.trace "Converts 'process' ${methodCall.arguments}" + + assert methodCall.arguments instanceof ArgumentListExpression + def list = (methodCall.arguments as ArgumentListExpression).getExpressions() + + // extract the first argument which has to be a method-call expression + // the name of this method represent the *process* name + if( list.size() != 1 || !list[0].class.isAssignableFrom(MethodCallExpression) ) { + log.debug "Missing name in process definition at line: ${methodCall.lineNumber}" + unit.addError( new SyntaxException("Process definition syntax error -- A string identifier must be provided after the `process` keyword", methodCall.lineNumber, methodCall.columnNumber+7)) + return + } + + def nested = list[0] as MethodCallExpression + def name = nested.getMethodAsString() + // check the process name is not defined yet + if( isIllegalName(name, methodCall) ) { + return + } + processNames.add(name) + + // the nested method arguments are the arguments to be passed + // to the process definition, plus adding the process *name* + // as an extra item in the arguments list + def args = nested.getArguments() as ArgumentListExpression + log.trace "Process name: $name with args: $args" + + // make sure to add the 'name' after the map item + // (which represent the named parameter attributes) + list = args.getExpressions() + if( list.size()>0 && list[0] instanceof MapExpression ) { + list.add(1, new ConstantExpression(name)) + } + else { + list.add(0, new ConstantExpression(name)) + } + + // set the new list as the new arguments + methodCall.setArguments( args ) + + // now continue as before ! + convertProcessBlock(methodCall, unit) + } + + /** + * Fetch all the variable references in a closure expression. + * + * @param closure + * @param unit + * @return The set of variable names referenced in the script. NOTE: it includes properties in the form {@code object.propertyName} + */ + protected Set fetchVariables( ClosureExpression closure, SourceUnit unit ) { + def visitor = new VariableVisitor(unit) + visitor.visitClosureExpression(closure) + return visitor.allVariables + } + +} diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSL.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSL.groovy deleted file mode 100644 index 7d54cb0e8f..0000000000 --- a/modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSL.groovy +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2013-2023, Seqera Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package nextflow.ast - -import java.lang.annotation.ElementType -import java.lang.annotation.Retention -import java.lang.annotation.RetentionPolicy -import java.lang.annotation.Target - -import org.codehaus.groovy.transform.GroovyASTTransformationClass - -/** - * Marker interface which to apply AST transformation to {@code process} declaration - */ -@Retention(RetentionPolicy.SOURCE) -@Target(ElementType.METHOD) -@GroovyASTTransformationClass(classes = [NextflowDSLImpl]) -@interface NextflowDSL {} diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSLImpl.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSLImpl.groovy deleted file mode 100644 index a278fdd8f6..0000000000 --- a/modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSLImpl.groovy +++ /dev/null @@ -1,1272 +0,0 @@ -/* - * Copyright 2013-2023, Seqera Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package nextflow.ast - -import static nextflow.Const.* -import static nextflow.ast.ASTHelpers.* -import static org.codehaus.groovy.ast.tools.GeneralUtils.* - -import groovy.transform.CompileStatic -import groovy.util.logging.Slf4j -import nextflow.NF -import nextflow.script.BaseScript -import nextflow.script.BodyDef -import nextflow.script.IncludeDef -import nextflow.script.TaskClosure -import nextflow.script.TokenEnvCall -import nextflow.script.TokenFileCall -import nextflow.script.TokenPathCall -import nextflow.script.TokenStdinCall -import nextflow.script.TokenStdoutCall -import nextflow.script.TokenValCall -import nextflow.script.TokenValRef -import nextflow.script.TokenVar -import org.codehaus.groovy.ast.ASTNode -import org.codehaus.groovy.ast.ClassCodeVisitorSupport -import org.codehaus.groovy.ast.ClassNode -import org.codehaus.groovy.ast.MethodNode -import org.codehaus.groovy.ast.Parameter -import org.codehaus.groovy.ast.VariableScope -import org.codehaus.groovy.ast.expr.ArgumentListExpression -import org.codehaus.groovy.ast.expr.BinaryExpression -import org.codehaus.groovy.ast.expr.CastExpression -import org.codehaus.groovy.ast.expr.ClosureExpression -import org.codehaus.groovy.ast.expr.ConstantExpression -import org.codehaus.groovy.ast.expr.Expression -import org.codehaus.groovy.ast.expr.GStringExpression -import org.codehaus.groovy.ast.expr.ListExpression -import org.codehaus.groovy.ast.expr.MapEntryExpression -import org.codehaus.groovy.ast.expr.MapExpression -import org.codehaus.groovy.ast.expr.MethodCallExpression -import org.codehaus.groovy.ast.expr.PropertyExpression -import org.codehaus.groovy.ast.expr.TupleExpression -import org.codehaus.groovy.ast.expr.UnaryMinusExpression -import org.codehaus.groovy.ast.expr.VariableExpression -import org.codehaus.groovy.ast.stmt.BlockStatement -import org.codehaus.groovy.ast.stmt.ExpressionStatement -import org.codehaus.groovy.ast.stmt.ReturnStatement -import org.codehaus.groovy.ast.stmt.Statement -import org.codehaus.groovy.control.CompilePhase -import org.codehaus.groovy.control.SourceUnit -import org.codehaus.groovy.syntax.SyntaxException -import org.codehaus.groovy.syntax.Token -import org.codehaus.groovy.syntax.Types -import org.codehaus.groovy.transform.ASTTransformation -import org.codehaus.groovy.transform.GroovyASTTransformation -/** - * Implement some syntax sugars of Nextflow DSL scripting. - * - * @author Paolo Di Tommaso - */ - -@Slf4j -@CompileStatic -@GroovyASTTransformation(phase = CompilePhase.CONVERSION) -class NextflowDSLImpl implements ASTTransformation { - - final static private String WORKFLOW_TAKE = 'take' - final static private String WORKFLOW_EMIT = 'emit' - final static private String WORKFLOW_MAIN = 'main' - final static private List SCOPES = [WORKFLOW_TAKE, WORKFLOW_EMIT, WORKFLOW_MAIN] - - final static public String PROCESS_WHEN = 'when' - final static public String PROCESS_STUB = 'stub' - - static public String OUT_PREFIX = '$out' - - static private Set RESERVED_NAMES - - static { - // method names implicitly defined by the groovy script SHELL - RESERVED_NAMES = ['main','run','runScript'] as Set - // existing method cannot be used for custom script definition - for( def method : BaseScript.getMethods() ) { - RESERVED_NAMES.add(method.name) - } - - } - - @Override - void visit(ASTNode[] astNodes, SourceUnit unit) { - createVisitor(unit).visitClass((ClassNode)astNodes[1]) - } - - /* - * create the code visitor - */ - protected ClassCodeVisitorSupport createVisitor( SourceUnit unit ) { - new DslCodeVisitor(unit) - } - - @CompileStatic - static class DslCodeVisitor extends ClassCodeVisitorSupport { - - - final private SourceUnit unit - - private String currentTaskName - - private String currentLabel - - private String bodyLabel - - private Set processNames = [] - - private Set workflowNames = [] - - private Set functionNames = [] - - private int anonymousWorkflow - - protected SourceUnit getSourceUnit() { unit } - - - DslCodeVisitor(SourceUnit unit) { - this.unit = unit - } - - @Override - void visitMethod(MethodNode node) { - if( node.public && !node.static && !node.synthetic && !node.metaDataMap?.'org.codehaus.groovy.ast.MethodNode.isScriptBody') { - if( !isIllegalName(node.name, node)) - functionNames.add(node.name) - } - super.visitMethod(node) - } - - @Override - void visitMethodCallExpression(MethodCallExpression methodCall) { - // pre-condition to be verified to apply the transformation - final preCondition = methodCall.objectExpression?.getText() == 'this' - final methodName = methodCall.getMethodAsString() - - /* - * intercept the *process* method in order to transform the script closure - */ - if( methodName == 'process' && preCondition ) { - - // clear block label - bodyLabel = null - currentLabel = null - currentTaskName = methodName - try { - convertProcessDef(methodCall,sourceUnit) - super.visitMethodCallExpression(methodCall) - } - finally { - currentTaskName = null - } - } - else if( methodName == 'workflow' && preCondition ) { - convertWorkflowDef(methodCall,sourceUnit) - super.visitMethodCallExpression(methodCall) - } - - // just apply the default behavior - else { - super.visitMethodCallExpression(methodCall) - } - - } - - @Override - void visitExpressionStatement(ExpressionStatement stm) { - if( stm.text.startsWith('this.include(') && stm.getExpression() instanceof MethodCallExpression ) { - final methodCall = (MethodCallExpression)stm.getExpression() - convertIncludeDef(methodCall) - // this is necessary to invoke the `load` method on the include definition - final loadCall = new MethodCallExpression(methodCall, 'load0', new ArgumentListExpression(new VariableExpression('params'))) - stm.setExpression(loadCall) - } - super.visitExpressionStatement(stm) - } - - protected void convertIncludeDef(MethodCallExpression call) { - if( call.methodAsString=='include' && call.arguments instanceof ArgumentListExpression ) { - final allArgs = (ArgumentListExpression)call.arguments - if( allArgs.size() != 1 ) { - syntaxError(call, "Not a valid include definition -- it must specify the module path") - return - } - - final arg = allArgs[0] - final newArgs = new ArgumentListExpression() - if( arg instanceof ConstantExpression ) { - newArgs.addExpression( createX(IncludeDef, arg) ) - } - else if( arg instanceof VariableExpression ) { - // the name of the component i.e. process, workflow, etc to import - final component = arg.getName() - // wrap the name in a `TokenVar` type - final token = createX(TokenVar, new ConstantExpression(component)) - // create a new `IncludeDef` object - newArgs.addExpression(createX(IncludeDef, token)) - } - else if( arg instanceof CastExpression && arg.getExpression() instanceof VariableExpression) { - def cast = (CastExpression)arg - // the name of the component i.e. process, workflow, etc to import - final component = (cast.expression as VariableExpression).getName() - // wrap the name in a `TokenVar` type - final token = createX(TokenVar, new ConstantExpression(component)) - // the alias to give it - final alias = constX(cast.type.name) - newArgs.addExpression( createX(IncludeDef, token, alias) ) - } - else if( arg instanceof ClosureExpression ) { - // multiple modules inclusion - final block = (BlockStatement)arg.getCode() - final modulesList = new ListExpression() - for( Statement stm : block.statements ) { - if( stm instanceof ExpressionStatement ) { - CastExpression castX - VariableExpression varX - Expression moduleX - if( (varX=isVariableX(stm.expression)) ) { - def name = constX(varX.name) - moduleX = createX(IncludeDef.Module, name) - } - else if( (castX=isCastX(stm.expression)) && (varX=isVariableX(castX.expression)) ) { - def name = constX(varX.name) - final alias = constX(castX.type.name) - moduleX = createX(IncludeDef.Module, name, alias) - } - else { - syntaxError(call, "Not a valid include module name") - return - } - modulesList.addExpression(moduleX) - } - else { - syntaxError(call, "Not a valid include module name") - return - } - - } - newArgs.addExpression( createX(IncludeDef, modulesList) ) - } - else { - syntaxError(call, "Not a valid include definition -- it must specify the module path as a string") - return - } - call.setArguments(newArgs) - } - else if( call.objectExpression instanceof MethodCallExpression ) { - convertIncludeDef((MethodCallExpression)call.objectExpression) - } - } - - /* - * this method transforms the DSL definition - * - * workflow foo { - * code - * } - * - * into a method invocation as - * - * workflow('foo', { -> code }) - * - */ - protected void convertWorkflowDef(MethodCallExpression methodCall, SourceUnit unit) { - log.trace "Convert 'workflow' ${methodCall.arguments}" - - assert methodCall.arguments instanceof ArgumentListExpression - def args = (ArgumentListExpression)methodCall.arguments - def len = args.size() - - // anonymous workflow definition - if( len == 1 && args[0] instanceof ClosureExpression ) { - if( anonymousWorkflow++ > 0 ) { - unit.addError( new SyntaxException("Duplicate entry workflow definition", methodCall.lineNumber, methodCall.columnNumber+8)) - return - } - - def newArgs = new ArgumentListExpression() - def body = (ClosureExpression)args[0] - newArgs.addExpression( makeWorkflowDefWrapper(body,true) ) - methodCall.setArguments( newArgs ) - return - } - - // extract the first argument which has to be a method-call expression - // the name of this method represent the *workflow* name - if( len != 1 || !args[0].class.isAssignableFrom(MethodCallExpression) ) { - log.debug "Missing name in workflow definition at line: ${methodCall.lineNumber}" - unit.addError( new SyntaxException("Workflow definition syntax error -- A string identifier must be provided after the `workflow` keyword", methodCall.lineNumber, methodCall.columnNumber+8)) - return - } - - final nested = args[0] as MethodCallExpression - final name = nested.getMethodAsString() - // check the process name is not defined yet - if( isIllegalName(name, methodCall) ) { - return - } - workflowNames.add(name) - - // the nested method arguments are the arguments to be passed - // to the process definition, plus adding the process *name* - // as an extra item in the arguments list - args = (ArgumentListExpression)nested.getArguments() - len = args.size() - log.trace "Workflow name: $name with args: $args" - - // make sure to add the 'name' after the map item - // (which represent the named parameter attributes) - def newArgs = new ArgumentListExpression() - - // add the workflow body def - if( len != 1 || !(args[0] instanceof ClosureExpression)) { - syntaxError(methodCall, "Invalid workflow definition") - return - } - - final body = (ClosureExpression)args[0] - newArgs.addExpression( constX(name) ) - newArgs.addExpression( makeWorkflowDefWrapper(body,false) ) - - // set the new list as the new arguments - methodCall.setArguments( newArgs ) - } - - - protected Statement normWorkflowParam(ExpressionStatement stat, String type, Set uniqueNames, List body) { - MethodCallExpression callx - VariableExpression varx - - if( (callx=isMethodCallX(stat.expression)) && isThisX(callx.objectExpression) ) { - final name = "_${type}_${callx.methodAsString}" - return stmt( callThisX(name, callx.arguments) ) - } - - if( (varx=isVariableX(stat.expression)) ) { - final name = "_${type}_${varx.name}" - return stmt( callThisX(name) ) - } - - if( type == WORKFLOW_EMIT ) { - return createAssignX(stat, body, type, uniqueNames) - } - - syntaxError(stat, "Workflow malformed parameter definition") - return stat - } - - protected Statement createAssignX(ExpressionStatement stat, List body, String type, Set uniqueNames) { - BinaryExpression binx - MethodCallExpression callx - Expression args=null - - if( (binx=isAssignX(stat.expression)) ) { - // keep the statement in body to allow it to be evaluated - body.add(stat) - // and create method call expr to capture the var name in the emission - final left = (VariableExpression)binx.leftExpression - final name = "_${type}_${left.name}" - return stmt( callThisX(name) ) - } - - if( (callx=isMethodCallX(stat.expression)) && callx.objectExpression.text!='this' && hasTo(callx)) { - // keep the args - args = callx.arguments - // replace the method call expression with a property - stat.expression = new PropertyExpression(callx.objectExpression, callx.method) - // then, fallback to default case - } - - // wrap the expression into a assignment expression - final var = getNextName(uniqueNames) - final left = new VariableExpression(var) - final right = stat.expression - final token = new Token(Types.ASSIGN, '=', -1, -1) - final assign = new BinaryExpression(left, token, right) - body.add(stmt(assign)) - - // the call method statement for the emit declaration - final name="_${type}_${var}" - callx = args ? callThisX(name, args) : callThisX(name) - return stmt(callx) - } - - protected boolean hasTo(MethodCallExpression callX) { - def tupleX = isTupleX(callX.arguments) - if( !tupleX ) return false - if( !tupleX.expressions ) return false - def mapX = isMapX(tupleX.expressions[0]) - if( !mapX ) return false - def entry = mapX.getMapEntryExpressions().find { isConstX(it.keyExpression).text=='to' } - return entry != null - } - - protected String getNextName(Set allNames) { - String result - while( true ) { - result = OUT_PREFIX + allNames.size() - if( allNames.add(result) ) - break - } - return result - } - - protected Expression makeWorkflowDefWrapper( ClosureExpression closure, boolean anonymous ) { - - final codeBlock = (BlockStatement) closure.code - final codeStms = codeBlock.statements - final scope = codeBlock.variableScope - - final visited = new HashMap(5); - final emitNames = new LinkedHashSet(codeStms.size()) - final wrap = new ArrayList(codeStms.size()) - final body = new ArrayList(codeStms.size()) - final source = new StringBuilder() - String context = null - String previous = null - for( Statement stm : codeStms ) { - previous = context - context = stm.statementLabel ?: context - // check for changing context - if( context && context != previous ) { - if( visited[context] && visited[previous] ) { - syntaxError(stm, "Unexpected workflow `${context}` context here") - break - } - } - visited[context] = true - - switch (context) { - case WORKFLOW_TAKE: - case WORKFLOW_EMIT: - if( !(stm instanceof ExpressionStatement) ) { - syntaxError(stm, "Workflow malformed parameter definition") - break - } - wrap.add(normWorkflowParam(stm as ExpressionStatement, context, emitNames, body)) - break - - case WORKFLOW_MAIN: - body.add(stm) - break - - default: - if( context ) { - def opts = SCOPES.closest(context) - def msg = "Unknown execution scope '$context:'" - if( opts ) msg += " -- Did you mean ${opts.collect{"'$it'"}.join(', ')}" - syntaxError(stm, msg) - } - body.add(stm) - } - } - // read the closure source - readSource(closure, source, unit, true) - - final bodyClosure = closureX(null, block(scope, body)) - final invokeBody = makeScriptWrapper(bodyClosure, source.toString(), 'workflow', unit) - wrap.add( stmt(invokeBody) ) - - closureX(null, block(scope, wrap)) - } - - protected void syntaxError(ASTNode node, String message) { - int line = node.lineNumber - int coln = node.columnNumber - unit.addError( new SyntaxException(message,line,coln)) - } - - /** - * Transform a DSL `process` definition into a proper method invocation - * - * @param methodCall - * @param unit - */ - protected void convertProcessBlock( MethodCallExpression methodCall, SourceUnit unit ) { - log.trace "Apply task closure transformation to method call: $methodCall" - - final args = methodCall.arguments as ArgumentListExpression - final lastArg = args.expressions.size()>0 ? args.getExpression(args.expressions.size()-1) : null - final isClosure = lastArg instanceof ClosureExpression - - if( isClosure ) { - // the block holding all the statements defined in the process (closure) definition - final block = (lastArg as ClosureExpression).code as BlockStatement - - /* - * iterate over the list of statements to: - * - converts the method after the 'input:' label as input parameters - * - converts the method after the 'output:' label as output parameters - * - collect all the statement after the 'exec:' label - */ - def source = new StringBuilder() - List execStatements = [] - - List whenStatements = [] - def whenSource = new StringBuilder() - - List stubStatements = [] - def stubSource = new StringBuilder() - - - def iterator = block.getStatements().iterator() - while( iterator.hasNext() ) { - - // get next statement - Statement stm = iterator.next() - - // keep track of current block label - currentLabel = stm.statementLabel ?: currentLabel - - switch(currentLabel) { - case 'input': - if( stm instanceof ExpressionStatement ) { - fixLazyGString( stm ) - fixStdinStdout( stm ) - convertInputMethod( stm.getExpression() ) - } - break - - case 'output': - if( stm instanceof ExpressionStatement ) { - fixLazyGString( stm ) - fixStdinStdout( stm ) - convertOutputMethod( stm.getExpression() ) - } - break - - case 'exec': - bodyLabel = currentLabel - iterator.remove() - execStatements << stm - readSource(stm,source,unit) - break - - case 'script': - case 'shell': - bodyLabel = currentLabel - iterator.remove() - execStatements << stm - readSource(stm,source,unit) - break - - case PROCESS_STUB: - iterator.remove() - stubStatements << stm - readSource(stm,stubSource,unit) - break - - // capture the statements in a when guard and remove from the current block - case PROCESS_WHEN: - if( iterator.hasNext() ) { - iterator.remove() - whenStatements << stm - readSource(stm,whenSource,unit) - break - } - // when entering in this branch means that this is the last statement, - // which is supposed to be the task command - // hence if no previous `when` statement has been processed, a syntax error is returned - else if( !whenStatements ) { - int line = methodCall.lineNumber - int coln = methodCall.columnNumber - unit.addError(new SyntaxException("Invalid process definition -- Empty `when` or missing `script` statement", line, coln)) - return - } - else - break - - default: - if(currentLabel) { - def line = stm.getLineNumber() - def coln = stm.getColumnNumber() - unit.addError(new SyntaxException("Invalid process definition -- Unknown keyword `$currentLabel`",line,coln)) - return - } - - fixLazyGString(stm) - fixDirectiveWithNegativeValue(stm) // Fixes #180 - } - } - - /* - * add the `when` block if found - */ - if( whenStatements ) { - addWhenGuardCall(whenStatements, whenSource, block) - } - - /* - * add try `stub` block if found - */ - if( stubStatements ) { - final newBLock = addStubCall(stubStatements, stubSource, block) - newBLock.visit(new TaskCmdXformVisitor(unit)) - } - - /* - * wrap all the statements after the 'exec:' label by a new closure containing them (in a new block) - */ - final len = block.statements.size() - boolean done = false - if( execStatements ) { - // create a new Closure - def execBlock = new BlockStatement(execStatements, new VariableScope(block.variableScope)) - def execClosure = new ClosureExpression( Parameter.EMPTY_ARRAY, execBlock ) - - // append the new block to the - // set the 'script' flag parameter - def wrap = makeScriptWrapper(execClosure, source, bodyLabel, unit) - block.addStatement( new ExpressionStatement(wrap) ) - if( bodyLabel == 'script' ) - block.visit(new TaskCmdXformVisitor(unit)) - done = true - - } - // when only the `stub` block is defined add an empty command - else if ( !bodyLabel && stubStatements ) { - final cmd = 'true' - final list = new ArrayList(1); - list.add( new ExpressionStatement(constX(cmd)) ) - final dummyBlock = new BlockStatement( list, new VariableScope(block.variableScope)) - final dummyClosure = new ClosureExpression( Parameter.EMPTY_ARRAY, dummyBlock ) - - // append the new block to the - // set the 'script' flag parameter - final wrap = makeScriptWrapper(dummyClosure, cmd, 'script', unit) - block.addStatement( new ExpressionStatement(wrap) ) - done = true - } - - /* - * when the last statement is a string script, the 'script:' label can be omitted - */ - else if( len ) { - def stm = block.getStatements().get(len-1) - readSource(stm,source,unit) - - if ( stm instanceof ReturnStatement ){ - done = wrapExpressionWithClosure(block, stm.getExpression(), len, source, unit) - } - - else if ( stm instanceof ExpressionStatement ) { - done = wrapExpressionWithClosure(block, stm.getExpression(), len, source, unit) - } - - // apply command variables escape - stm.visit(new TaskCmdXformVisitor(unit)) - } - - if (!done) { - log.trace "Invalid 'process' definition -- Process must terminate with string expression" - int line = methodCall.lineNumber - int coln = methodCall.columnNumber - unit.addError( new SyntaxException("Invalid process definition -- Make sure the process ends with a script wrapped by quote characters",line,coln)) - } - } - } - - /** - * Converts a `when` block into a when method call expression. The when code is converted into a - * closure expression and set a `when` directive in the process configuration properties. - * - * See {@link nextflow.script.ProcessConfig#configProperties} - * See {@link nextflow.processor.TaskConfig#getGuard(java.lang.String)} - */ - protected BlockStatement addWhenGuardCall( List statements, StringBuilder source, BlockStatement parent ) { - createBlock0(PROCESS_WHEN, statements, source, parent) - } - - protected BlockStatement addStubCall(List statements, StringBuilder source, BlockStatement parent ) { - createBlock0(PROCESS_STUB, statements, source, parent) - } - - protected BlockStatement createBlock0( String blockName, List statements, StringBuilder source, BlockStatement parent ) { - // wrap the code block into a closure expression - def block = new BlockStatement(statements, new VariableScope(parent.variableScope)) - def closure = new ClosureExpression( Parameter.EMPTY_ARRAY, block ) - - // the closure expression is wrapped itself into a TaskClosure object - // in order to capture the closure source other than the closure code - List newArgs = [] - newArgs << closure - newArgs << new ConstantExpression(source.toString()) - def whenObj = createX( TaskClosure, newArgs ) - - // creates a method call expression for the method `when` - def method = new MethodCallExpression(VariableExpression.THIS_EXPRESSION, blockName, whenObj) - parent.getStatements().add(0, new ExpressionStatement(method)) - - return block - } - - /** - * Wrap the user provided piece of code, either a script or a closure with a {@code BodyDef} object - * - * @param closure - * @param source - * @param scriptOrNative - * @param unit - * @return a {@code BodyDef} object - */ - private Expression makeScriptWrapper( ClosureExpression closure, CharSequence source, String section, SourceUnit unit ) { - - final List newArgs = [] - newArgs << (closure) - newArgs << ( new ConstantExpression(source.toString()) ) - newArgs << ( new ConstantExpression(section) ) - - // collect all variable tokens and pass them as single list argument - final variables = fetchVariables(closure,unit) - final listArg = new ArrayList(variables.size()) - for( TokenValRef var: variables ) { - def pName = new ConstantExpression(var.name) - def pLine = new ConstantExpression(var.lineNum) - def pCol = new ConstantExpression(var.colNum) - listArg << createX( TokenValRef, pName, pLine, pCol ) - } - newArgs << ( new ListExpression(listArg) ) - - // invokes the BodyDef constructor - createX( BodyDef, newArgs ) - } - - /** - * Read the user provided script source string - * - * @param node - * @param buffer - * @param unit - */ - private void readSource( ASTNode node, StringBuilder buffer, SourceUnit unit, stripBrackets=false ) { - final colx = node.getColumnNumber() - final colz = node.getLastColumnNumber() - final first = node.getLineNumber() - final last = node.getLastLineNumber() - for( int i=first; i<=last; i++ ) { - def line = unit.source.getLine(i, null) - if( i==last ) { - line = line.substring(0,colz-1) - if( stripBrackets ) { - line = line.replaceFirst(/}.*$/,'') - if( !line.trim() ) continue - } - } - if( i==first ) { - line = line.substring(colx-1) - if( stripBrackets ) { - line = line.replaceFirst(/^.*\{/,'').trim() - if( !line.trim() ) continue - } - } - buffer.append(line) .append('\n') - } - } - - protected void fixLazyGString( Statement stm ) { - if( stm instanceof ExpressionStatement && stm.getExpression() instanceof MethodCallExpression ) { - new GStringToLazyVisitor(unit).visitExpressionStatement(stm) - } - } - - protected void fixDirectiveWithNegativeValue( Statement stm ) { - if( stm instanceof ExpressionStatement && stm.getExpression() instanceof BinaryExpression ) { - def binary = (BinaryExpression)stm.getExpression() - if(!(binary.leftExpression instanceof VariableExpression)) - return - if( binary.operation.type != Types.MINUS ) - return - - // -- transform the binary expression into a method call expression - // where the left expression represents the method name to invoke - def methodName = ((VariableExpression)binary.leftExpression).name - - // -- wrap the value into a minus operator - def value = (Expression)new UnaryMinusExpression( binary.rightExpression ) - def args = new ArgumentListExpression( [value] ) - - // -- create the method call expression and replace it to the binary expression - def call = new MethodCallExpression(new VariableExpression('this'), methodName, args) - stm.setExpression(call) - - } - } - - protected void fixStdinStdout( ExpressionStatement stm ) { - - // transform the following syntax: - // `stdin from x` --> stdin() from (x) - // `stdout into x` --> `stdout() into (x)` - VariableExpression varX - if( stm.expression instanceof PropertyExpression ) { - def expr = (PropertyExpression)stm.expression - def obj = expr.objectExpression - def prop = expr.property as ConstantExpression - def target = new VariableExpression(prop.text) - - if( obj instanceof MethodCallExpression ) { - def methodCall = obj as MethodCallExpression - if( 'stdout' == methodCall.getMethodAsString() ) { - def stdout = new MethodCallExpression( new VariableExpression('this'), 'stdout', new ArgumentListExpression() ) - def into = new MethodCallExpression(stdout, 'into', new ArgumentListExpression(target)) - // remove replace the old one with the new one - stm.setExpression( into ) - } - else if( 'stdin' == methodCall.getMethodAsString() ) { - def stdin = new MethodCallExpression( new VariableExpression('this'), 'stdin', new ArgumentListExpression() ) - def from = new MethodCallExpression(stdin, 'from', new ArgumentListExpression(target)) - // remove replace the old one with the new one - stm.setExpression( from ) - } - } - } - // transform the following syntax: - // `stdout into (x,y,..)` --> `stdout() into (x,y,..)` - else if( stm.expression instanceof MethodCallExpression ) { - def methodCall = (MethodCallExpression)stm.expression - if( 'stdout' == methodCall.getMethodAsString() ) { - def args = methodCall.getArguments() - if( args instanceof ArgumentListExpression && args.getExpressions() && args.getExpression(0) instanceof MethodCallExpression ) { - def methodCall2 = (MethodCallExpression)args.getExpression(0) - def args2 = methodCall2.getArguments() - if( args2 instanceof ArgumentListExpression && methodCall2.methodAsString == 'into') { - def vars = args2.getExpressions() - def stdout = new MethodCallExpression( new VariableExpression('this'), 'stdout', new ArgumentListExpression() ) - def into = new MethodCallExpression(stdout, 'into', new ArgumentListExpression(vars)) - // remove replace the old one with the new one - stm.setExpression( into ) - } - } - } - } - else if( (varX=isVariableX(stm.expression)) && (varX.name=='stdin' || varX.name=='stdout') && NF.isDsl2() ) { - final name = varX.name=='stdin' ? '_in_stdin' : '_out_stdout' - final call = new MethodCallExpression( new VariableExpression('this'), name, new ArgumentListExpression() ) - // remove replace the old one with the new one - stm.setExpression(call) - } - } - - /* - * handle *input* parameters - */ - protected void convertInputMethod( Expression expression ) { - log.trace "convert > input expression: $expression" - - if( expression instanceof MethodCallExpression ) { - - def methodCall = expression as MethodCallExpression - def methodName = methodCall.getMethodAsString() - def nested = methodCall.objectExpression instanceof MethodCallExpression - log.trace "convert > input method: $methodName" - - if( methodName in ['val','env','file','each','set','stdin','path','tuple'] ) { - //this methods require a special prefix - if( !nested ) - methodCall.setMethod( new ConstantExpression('_in_' + methodName) ) - - fixMethodCall(methodCall) - } - - /* - * Handles a GString a file name, like this: - * - * input: - * file x name "$var_name" from q - * - */ - else if( methodName == 'name' && isWithinMethod(expression, 'file') ) { - varToConstX(methodCall.getArguments()) - } - - // invoke on the next method call - if( expression.objectExpression instanceof MethodCallExpression ) { - convertInputMethod(methodCall.objectExpression) - } - } - - else if( expression instanceof PropertyExpression ) { - // invoke on the next method call - if( expression.objectExpression instanceof MethodCallExpression ) { - convertInputMethod(expression.objectExpression) - } - } - - } - - protected boolean isWithinMethod(MethodCallExpression method, String name) { - if( method.objectExpression instanceof MethodCallExpression ) { - return isWithinMethod(method.objectExpression as MethodCallExpression, name) - } - - return method.getMethodAsString() == name - } - - /** - * Transform a map entry `emit: something` into `emit: 'something' - * (ie. as a constant) in a map expression passed as argument to - * a method call. This allow the syntax - * - * output: - * path 'foo', emit: bar - * - * @param call - */ - protected void fixOutEmitOption(MethodCallExpression call) { - List args = isTupleX(call.arguments)?.expressions - if( !args ) return - if( args.size()<2 && (args.size()!=1 || call.methodAsString!='_out_stdout')) return - MapExpression map = isMapX(args[0]) - if( !map ) return - for( int i=0; i output expression: $expression" - - if( !(expression instanceof MethodCallExpression) ) { - return - } - - def methodCall = expression as MethodCallExpression - def methodName = methodCall.getMethodAsString() - def nested = methodCall.objectExpression instanceof MethodCallExpression - log.trace "convert > output method: $methodName" - - if( methodName in ['val','env','file','set','stdout','path','tuple'] && !nested ) { - // prefix the method name with the string '_out_' - methodCall.setMethod( new ConstantExpression('_out_' + methodName) ) - fixMethodCall(methodCall) - fixOutEmitOption(methodCall) - } - - else if( methodName in ['into','mode'] ) { - fixMethodCall(methodCall) - } - - // continue to traverse - if( methodCall.objectExpression instanceof MethodCallExpression ) { - convertOutputMethod(methodCall.objectExpression) - } - - } - - private boolean withinTupleMethod - - private boolean withinEachMethod - - /** - * This method converts the a method call argument from a Variable to a Constant value - * so that it is possible to reference variable that not yet exist - * - * @param methodCall The method object for which it is required to change args definition - * @param flagVariable Whenever append a flag specified if the variable replacement has been applied - * @param index The index of the argument to modify - * @return - */ - protected void fixMethodCall( MethodCallExpression methodCall ) { - final name = methodCall.methodAsString - - withinTupleMethod = name == '_in_set' || name == '_out_set' || name == '_in_tuple' || name == '_out_tuple' - withinEachMethod = name == '_in_each' - - try { - if( isOutputWithPropertyExpression(methodCall) ) { - // transform an output value declaration such - // output: val( obj.foo ) - // to - // output: val({ obj.foo }) - wrapPropertyToClosure((ArgumentListExpression)methodCall.getArguments()) - } - else - varToConstX(methodCall.getArguments()) - - } finally { - withinTupleMethod = false - withinEachMethod = false - } - } - - static final private List OUT_PROPERTY_VALID_TYPES = ['_out_val', '_out_env', '_out_file', '_out_path'] - - protected boolean isOutputWithPropertyExpression(MethodCallExpression methodCall) { - if( methodCall.methodAsString !in OUT_PROPERTY_VALID_TYPES ) - return false - if( methodCall.getArguments() instanceof ArgumentListExpression ) { - def args = (ArgumentListExpression)methodCall.getArguments() - if( args.size()==0 || args.size()>2 ) - return false - - return args.last() instanceof PropertyExpression - } - - return false - } - - protected void wrapPropertyToClosure(ArgumentListExpression expr) { - final args = expr as ArgumentListExpression - final property = (PropertyExpression) args.last() - final closure = wrapPropertyToClosure(property) - args.getExpressions().set(args.size()-1, closure) - } - - protected ClosureExpression wrapPropertyToClosure(PropertyExpression property) { - def block = new BlockStatement() - block.addStatement( new ExpressionStatement(property) ) - - def closure = new ClosureExpression( Parameter.EMPTY_ARRAY, block ) - closure.variableScope = new VariableScope(block.variableScope) - - return closure - } - - - protected Expression varToStrX( Expression expr ) { - if( expr instanceof VariableExpression ) { - def name = ((VariableExpression) expr).getName() - return createX( TokenVar, new ConstantExpression(name) ) - } - else if( expr instanceof PropertyExpression ) { - // transform an output declaration such - // output: tuple val( obj.foo ) - // to - // output: tuple val({ obj.foo }) - return wrapPropertyToClosure(expr) - } - - if( expr instanceof TupleExpression ) { - def i = 0 - def list = expr.getExpressions() - for( Expression item : list ) { - list[i++] = varToStrX(item) - } - - return expr - } - - return expr - } - - protected Expression varToConstX( Expression expr ) { - - if( expr instanceof VariableExpression ) { - // when it is a variable expression, replace it with a constant representing - // the variable name - def name = ((VariableExpression) expr).getName() - - /* - * the 'stdin' is used as placeholder for the standard input in the tuple definition. For example: - * - * input: - * tuple( stdin, .. ) from q - */ - if( name == 'stdin' && withinTupleMethod ) - return createX( TokenStdinCall ) - - /* - * input: - * tuple( stdout, .. ) - */ - else if ( name == 'stdout' && withinTupleMethod ) - return createX( TokenStdoutCall ) - - else - return createX( TokenVar, new ConstantExpression(name) ) - } - - if( expr instanceof MethodCallExpression ) { - def methodCall = expr as MethodCallExpression - - /* - * replace 'file' method call in the tuple definition, for example: - * - * input: - * tuple( file(fasta:'*.fa'), .. ) from q - */ - if( methodCall.methodAsString == 'file' && (withinTupleMethod || withinEachMethod) ) { - def args = (TupleExpression) varToConstX(methodCall.arguments) - return createX( TokenFileCall, args ) - } - else if( methodCall.methodAsString == 'path' && (withinTupleMethod || withinEachMethod) ) { - def args = (TupleExpression) varToConstX(methodCall.arguments) - return createX( TokenPathCall, args ) - } - - /* - * input: - * tuple( env(VAR_NAME) ) from q - */ - if( methodCall.methodAsString == 'env' && withinTupleMethod ) { - def args = (TupleExpression) varToStrX(methodCall.arguments) - return createX( TokenEnvCall, args ) - } - - /* - * input: - * tuple val(x), .. from q - */ - if( methodCall.methodAsString == 'val' && withinTupleMethod ) { - def args = (TupleExpression) varToStrX(methodCall.arguments) - return createX( TokenValCall, args ) - } - - } - - // -- TupleExpression or ArgumentListExpression - if( expr instanceof TupleExpression ) { - def i = 0 - def list = expr.getExpressions() - for( Expression item : list ) { - list[i++] = varToConstX(item) - } - return expr - } - - return expr - } - - /** - * Wrap a generic expression with in a closure expression - * - * @param block The block to which the resulting closure has to be appended - * @param expr The expression to the wrapped in a closure - * @param len - * @return A tuple in which: - *
  • 1st item: {@code true} if successful or {@code false} otherwise - *
  • 2nd item: on error condition the line containing the error in the source script, zero otherwise - *
  • 3rd item: on error condition the column containing the error in the source script, zero otherwise - * - */ - protected boolean wrapExpressionWithClosure( BlockStatement block, Expression expr, int len, CharSequence source, SourceUnit unit ) { - if( expr instanceof GStringExpression || expr instanceof ConstantExpression ) { - // remove the last expression - block.statements.remove(len-1) - - // and replace it by a wrapping closure - def closureExp = new ClosureExpression( Parameter.EMPTY_ARRAY, new ExpressionStatement(expr) ) - closureExp.variableScope = new VariableScope(block.variableScope) - - // append to the list of statement - //def wrap = newObj(BodyDef, closureExp, new ConstantExpression(source.toString()), ConstantExpression.TRUE) - def wrap = makeScriptWrapper(closureExp, source, 'script', unit ) - block.statements.add( new ExpressionStatement(wrap) ) - - return true - } - else if( expr instanceof ClosureExpression ) { - // do not touch it - return true - } - else { - log.trace "Invalid process result expression: ${expr} -- Only constant or string expression can be used" - } - - return false - } - - protected boolean isIllegalName(String name, ASTNode node) { - if( name in RESERVED_NAMES ) { - unit.addError( new SyntaxException("Identifier `$name` is reserved for internal use", node.lineNumber, node.columnNumber+8) ) - return true - } - if( name in workflowNames || name in processNames ) { - unit.addError( new SyntaxException("Identifier `$name` is already used by another definition", node.lineNumber, node.columnNumber+8) ) - return true - } - if( name.contains(SCOPE_SEP) ) { - def offset = 8+2+ name.indexOf(SCOPE_SEP) - unit.addError( new SyntaxException("Process and workflow names cannot contain colon character", node.lineNumber, node.columnNumber+offset) ) - return true - } - return false - } - - /** - * This method handle the process definition, so that it transform the user entered syntax - * process myName ( named: args, .. ) { code .. } - * - * into - * process ( [named:args,..], String myName ) { } - * - * @param methodCall - * @param unit - */ - protected void convertProcessDef( MethodCallExpression methodCall, SourceUnit unit ) { - log.trace "Converts 'process' ${methodCall.arguments}" - - assert methodCall.arguments instanceof ArgumentListExpression - def list = (methodCall.arguments as ArgumentListExpression).getExpressions() - - // extract the first argument which has to be a method-call expression - // the name of this method represent the *process* name - if( list.size() != 1 || !list[0].class.isAssignableFrom(MethodCallExpression) ) { - log.debug "Missing name in process definition at line: ${methodCall.lineNumber}" - unit.addError( new SyntaxException("Process definition syntax error -- A string identifier must be provided after the `process` keyword", methodCall.lineNumber, methodCall.columnNumber+7)) - return - } - - def nested = list[0] as MethodCallExpression - def name = nested.getMethodAsString() - // check the process name is not defined yet - if( isIllegalName(name, methodCall) ) { - return - } - processNames.add(name) - - // the nested method arguments are the arguments to be passed - // to the process definition, plus adding the process *name* - // as an extra item in the arguments list - def args = nested.getArguments() as ArgumentListExpression - log.trace "Process name: $name with args: $args" - - // make sure to add the 'name' after the map item - // (which represent the named parameter attributes) - list = args.getExpressions() - if( list.size()>0 && list[0] instanceof MapExpression ) { - list.add(1, new ConstantExpression(name)) - } - else { - list.add(0, new ConstantExpression(name)) - } - - // set the new list as the new arguments - methodCall.setArguments( args ) - - // now continue as before ! - convertProcessBlock(methodCall, unit) - } - - /** - * Fetch all the variable references in a closure expression. - * - * @param closure - * @param unit - * @return The set of variable names referenced in the script. NOTE: it includes properties in the form {@code object.propertyName} - */ - protected Set fetchVariables( ClosureExpression closure, SourceUnit unit ) { - def visitor = new VariableVisitor(unit) - visitor.visitClosureExpression(closure) - return visitor.allVariables - } - - } - -} diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/NextflowXformImpl.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/NextflowXformImpl.groovy index e4359118d3..fb88728ae6 100644 --- a/modules/nextflow/src/main/groovy/nextflow/ast/NextflowXformImpl.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/ast/NextflowXformImpl.groovy @@ -32,8 +32,7 @@ import org.codehaus.groovy.control.SourceUnit import org.codehaus.groovy.transform.ASTTransformation import org.codehaus.groovy.transform.GroovyASTTransformation /** - * Implements Nextflow Xform logic - * See http://groovy-lang.org/metaprogramming.html#_classcodeexpressiontransformer + * Implements the syntax transformations for Nextflow scripts. * * @author Paolo Di Tommaso */ @@ -42,91 +41,12 @@ import org.codehaus.groovy.transform.GroovyASTTransformation @GroovyASTTransformation(phase = CompilePhase.CONVERSION) class NextflowXformImpl implements ASTTransformation { - SourceUnit unit - @Override - void visit(ASTNode[] nodes, SourceUnit source) { - this.unit = unit - createVisitor().visitClass((ClassNode)nodes[1]) - } - - protected ClassCodeExpressionTransformer createVisitor() { - - new ClassCodeExpressionTransformer() { - - protected SourceUnit getSourceUnit() { unit } - - @Override - Expression transform(Expression expr) { - if (expr == null) - return null - - def newExpr = transformBinaryExpression(expr) - if( newExpr ) { - return newExpr - } - else if( expr instanceof ClosureExpression) { - visitClosureExpression(expr) - } - - return super.transform(expr) - } - - /** - * This method replaces the `==` with the invocation of - * {@link LangHelpers#compareEqual(java.lang.Object, java.lang.Object)} - * - * This is required to allow the comparisons of `Path` objects - * which by default are not supported because it implements the Comparator interface - * - * See - * {@link LangHelpers#compareEqual(java.lang.Object, java.lang.Object)} - * https://stackoverflow.com/questions/28355773/in-groovy-why-does-the-behaviour-of-change-for-interfaces-extending-compar#comment45123447_28387391 - * - */ - protected Expression transformBinaryExpression(Expression expr) { - - if( expr.class != BinaryExpression ) - return null - - def binary = expr as BinaryExpression - def left = binary.getLeftExpression() - def right = binary.getRightExpression() - - if( '=='.equals(binary.operation.text) ) - return call('compareEqual',left,right) - - if( '!='.equals(binary.operation.text) ) - return new NotExpression(call('compareEqual',left,right)) - - if( '<'.equals(binary.operation.text) ) - return call('compareLessThan', left,right) - - if( '<='.equals(binary.operation.text) ) - return call('compareLessThanEqual', left,right) - - if( '>'.equals(binary.operation.text) ) - return call('compareGreaterThan', left,right) - - if( '>='.equals(binary.operation.text) ) - return call('compareGreaterThanEqual', left,right) - - return null - } - - - private MethodCallExpression call(String method, Expression left, Expression right) { - - final a = transformBinaryExpression(left) ?: left - final b = transformBinaryExpression(right) ?: right - - GeneralUtils.callX( - GeneralUtils.classX(LangHelpers), - method, - GeneralUtils.args(a,b)) - } - - } + void visit(ASTNode[] nodes, SourceUnit unit) { + final classNode = (ClassNode)nodes[1] + new BinaryExpressionXform(unit).visitClass(classNode) + new DslCodeVisitor(unit).visitClass(classNode) + new OperatorXform(unit).visitClass(classNode) } } diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/OpXform.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/OpXform.groovy deleted file mode 100644 index 76c6d86ba6..0000000000 --- a/modules/nextflow/src/main/groovy/nextflow/ast/OpXform.groovy +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2013-2023, Seqera Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package nextflow.ast - -import java.lang.annotation.ElementType -import java.lang.annotation.Retention -import java.lang.annotation.RetentionPolicy -import java.lang.annotation.Target - -import org.codehaus.groovy.transform.GroovyASTTransformationClass - -/** - * Declares Nextflow operators AST xforms - */ -@Retention(RetentionPolicy.SOURCE) -@Target(ElementType.METHOD) -@GroovyASTTransformationClass(classes = [OpXformImpl]) -@interface OpXform {} diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/OpXformImpl.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/OperatorXform.groovy similarity index 90% rename from modules/nextflow/src/main/groovy/nextflow/ast/OpXformImpl.groovy rename to modules/nextflow/src/main/groovy/nextflow/ast/OperatorXform.groovy index c60a2a341e..8692fc01fc 100644 --- a/modules/nextflow/src/main/groovy/nextflow/ast/OpXformImpl.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/ast/OperatorXform.groovy @@ -60,7 +60,7 @@ import static org.codehaus.groovy.ast.tools.GeneralUtils.returnS import static org.codehaus.groovy.ast.tools.GeneralUtils.stmt import static org.codehaus.groovy.ast.tools.GeneralUtils.varX /** - * Implements Nextflow operators xform logic + * Implements the syntax transformations for Nextflow operators. * * See http://groovy-lang.org/metaprogramming.html#_classcodeexpressiontransformer * @@ -68,18 +68,72 @@ import static org.codehaus.groovy.ast.tools.GeneralUtils.varX */ @Slf4j @CompileStatic -@GroovyASTTransformation(phase = CompilePhase.CONVERSION) -class OpXformImpl implements ASTTransformation { +class OperatorXform extends ClassCodeExpressionTransformer { - static final public String BRANCH_METHOD_NAME = 'branch' + static final String BRANCH_METHOD_NAME = 'branch' - static final public String BRANCH_CRITERIA_FUN = 'branchCriteria' + static final String BRANCH_CRITERIA_FUN = 'branchCriteria' - static final public String MULTIMAP_METHOD_NAME = 'multiMap' + static final String MULTIMAP_METHOD_NAME = 'multiMap' - static final public String MULTIMAP_CRITERIA_FUN = 'multiMapCriteria' + static final String MULTIMAP_CRITERIA_FUN = 'multiMapCriteria' - SourceUnit unit + private final SourceUnit unit + + OperatorXform(SourceUnit unit) { + this.unit = unit + } + + @Override + protected SourceUnit getSourceUnit() { unit } + + @Override + Expression transform(Expression expr) { + if (expr == null) + return null + + ClosureExpression body + if( (body=isBranchOpCall(expr)) ) { + return new BranchTransformer(expr as MethodCallExpression, body).apply() + } + else if( (body=isMultiMapOpCall(expr)) ) { + return new MultiMapTransformer(expr as MethodCallExpression, body).apply() + } + else if( expr instanceof ClosureExpression) { + visitClosureExpression(expr) + } + + return super.transform(expr) + } + + protected ClosureExpression isBranchOpCall(Expression expr) { + final m = ASTHelpers.isMethodCallX(expr) + if( m ) { + final name = m.methodAsString + final args = isArgsX(m.arguments) + final ClosureExpression ret = args && args.size()>0 ? isClosureX(args.last()) : null + if( name==BRANCH_METHOD_NAME && args.size()==1 ) + return ret + if( name==BRANCH_CRITERIA_FUN && args.size()==1 && m.objectExpression.text=='this') + return ret + } + return null + } + + protected ClosureExpression isMultiMapOpCall(Expression expr) { + final m = ASTHelpers.isMethodCallX(expr) + if( m ) { + final name = m.methodAsString + final args = isArgsX(m.arguments) + final ClosureExpression ret = args && args.size()>0 ? isClosureX(args.last()) : null + if( name==MULTIMAP_METHOD_NAME && args.size()==1 ) + return ret + if( name==MULTIMAP_CRITERIA_FUN && args.size()==1 && m.objectExpression.text=='this') + return ret + + } + return null + } static class BranchCondition { String label @@ -92,7 +146,6 @@ class OpXformImpl implements ASTTransformation { } } - @CompileStatic class BranchTransformer { final List impl = new ArrayList<>(20) @@ -351,72 +404,6 @@ class OpXformImpl implements ASTTransformation { } } - @Override - void visit(ASTNode[] nodes, SourceUnit source) { - this.unit = unit - createVisitor().visitClass((ClassNode)nodes[1]) - } - - protected ClosureExpression isBranchOpCall(Expression expr) { - final m = ASTHelpers.isMethodCallX(expr) - if( m ) { - final name = m.methodAsString - final args = isArgsX(m.arguments) - final ClosureExpression ret = args && args.size()>0 ? isClosureX(args.last()) : null - if( name==BRANCH_METHOD_NAME && args.size()==1 ) - return ret - if( name==BRANCH_CRITERIA_FUN && args.size()==1 && m.objectExpression.text=='this') - return ret - } - return null - } - - protected ClosureExpression isMultiMapOpCall(Expression expr) { - final m = ASTHelpers.isMethodCallX(expr) - if( m ) { - final name = m.methodAsString - final args = isArgsX(m.arguments) - final ClosureExpression ret = args && args.size()>0 ? isClosureX(args.last()) : null - if( name==MULTIMAP_METHOD_NAME && args.size()==1 ) - return ret - if( name==MULTIMAP_CRITERIA_FUN && args.size()==1 && m.objectExpression.text=='this') - return ret - - } - return null - } - - - /** - * Visit AST node to apply operator xforms - */ - protected ClassCodeExpressionTransformer createVisitor() { - - new ClassCodeExpressionTransformer() { - - protected SourceUnit getSourceUnit() { unit } - - @Override - Expression transform(Expression expr) { - if (expr == null) - return null - - ClosureExpression body - if( (body=isBranchOpCall(expr)) ) { - return new BranchTransformer(expr as MethodCallExpression, body).apply() - } - else if( (body=isMultiMapOpCall(expr)) ) { - return new MultiMapTransformer(expr as MethodCallExpression, body).apply() - } - else if( expr instanceof ClosureExpression) { - visitClosureExpression(expr) - } - - return super.transform(expr) - } - } - } - } diff --git a/modules/nextflow/src/main/groovy/nextflow/config/ConfigParser.groovy b/modules/nextflow/src/main/groovy/nextflow/config/ConfigParser.groovy index af05ba8687..4145727256 100644 --- a/modules/nextflow/src/main/groovy/nextflow/config/ConfigParser.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/config/ConfigParser.groovy @@ -21,7 +21,6 @@ import java.nio.file.Path import ch.artecat.grengine.Grengine import com.google.common.hash.Hashing import groovy.transform.PackageScope -import nextflow.ast.NextflowXform import nextflow.exception.ConfigParseException import nextflow.extension.Bolts import nextflow.file.FileHelper @@ -172,7 +171,6 @@ class ConfigParser { if( renderClosureAsString ) params.put('renderClosureAsString', true) config.addCompilationCustomizers(new ASTTransformationCustomizer(params, ConfigTransform)) - config.addCompilationCustomizers(new ASTTransformationCustomizer(NextflowXform)) // add implicit types def importCustomizer = new ImportCustomizer() importCustomizer.addImports( Duration.name ) diff --git a/modules/nextflow/src/main/groovy/nextflow/config/ConfigTransformImpl.groovy b/modules/nextflow/src/main/groovy/nextflow/config/ConfigTransformImpl.groovy index 08c928115b..c65ea5af20 100644 --- a/modules/nextflow/src/main/groovy/nextflow/config/ConfigTransformImpl.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/config/ConfigTransformImpl.groovy @@ -18,6 +18,7 @@ package nextflow.config import groovy.transform.CompileStatic import groovy.util.logging.Slf4j +import nextflow.ast.BinaryExpressionXform import org.codehaus.groovy.ast.ASTNode import org.codehaus.groovy.ast.AnnotationNode import org.codehaus.groovy.ast.ClassCodeVisitorSupport @@ -55,6 +56,7 @@ class ConfigTransformImpl implements ASTTransformation { // the following line is mostly an hack to pass a parameter to this xform instance this.renderClosureAsString = annot.getMember('renderClosureAsString') != null createVisitor(unit).visitClass(clazz) + new BinaryExpressionXform(unit).visitClass(clazz) } protected ClassCodeVisitorSupport createVisitor(SourceUnit unit) { diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskConfig.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskConfig.groovy index 3a24dc6c04..e7f0658148 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskConfig.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskConfig.groovy @@ -24,7 +24,7 @@ import java.nio.file.Path import groovy.transform.CompileStatic import nextflow.Const -import nextflow.ast.NextflowDSLImpl +import nextflow.ast.DslCodeVisitor import nextflow.exception.AbortOperationException import nextflow.exception.FailedGuardException import nextflow.executor.BashWrapperBuilder @@ -509,7 +509,7 @@ class TaskConfig extends LazyMap implements Cloneable { protected TaskClosure getStubBlock() { - final code = target.get(NextflowDSLImpl.PROCESS_STUB) + final code = target.get(DslCodeVisitor.PROCESS_STUB) if( !code ) return null if( code instanceof TaskClosure ) diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy index 69b95e689a..5b8dfa2774 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy @@ -52,7 +52,7 @@ import groovyx.gpars.group.PGroup import nextflow.NF import nextflow.Nextflow import nextflow.Session -import nextflow.ast.NextflowDSLImpl +import nextflow.ast.DslCodeVisitor import nextflow.ast.TaskCmdXform import nextflow.ast.TaskTemplateVarsXform import nextflow.cloud.CloudSpotTerminationException @@ -2254,7 +2254,7 @@ class TaskProcessor { protected boolean checkWhenGuard(TaskRun task) { try { - def pass = task.config.getGuard(NextflowDSLImpl.PROCESS_WHEN) + def pass = task.config.getGuard(DslCodeVisitor.PROCESS_WHEN) if( pass ) { return true } diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ChannelOut.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ChannelOut.groovy index 85beb3c65a..1d48e150c4 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/ChannelOut.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/ChannelOut.groovy @@ -19,10 +19,10 @@ package nextflow.script import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import groovyx.gpars.dataflow.DataflowWriteChannel +import nextflow.ast.DslCodeVisitor import nextflow.exception.DuplicateChannelNameException import nextflow.script.params.OutParam import nextflow.script.params.OutputsList -import static nextflow.ast.NextflowDSLImpl.OUT_PREFIX /** * Models the output of a process or a workflow component returning * more than one output channels @@ -67,7 +67,9 @@ class ChannelOut implements List { target = Collections.unmodifiableList(onlyWithName) } - Set getNames() { channels.keySet().findAll { !it.startsWith(OUT_PREFIX) } } + Set getNames() { + channels.keySet().findAll { !it.startsWith(DslCodeVisitor.OUT_PREFIX) } + } @Override def getProperty(String name) { diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessConfig.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessConfig.groovy index a9c2a86cf9..5848b9aa38 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessConfig.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessConfig.groovy @@ -22,9 +22,8 @@ import groovy.transform.PackageScope import groovy.util.logging.Slf4j import nextflow.Const import nextflow.NF -import nextflow.ast.NextflowDSLImpl +import nextflow.ast.DslCodeVisitor import nextflow.exception.ConfigParseException -import nextflow.exception.IllegalConfigException import nextflow.exception.IllegalDirectiveException import nextflow.executor.BashWrapperBuilder import nextflow.processor.ConfigList @@ -204,9 +203,9 @@ class ProcessConfig implements Map, Cloneable { private void checkName(String name) { if( DIRECTIVES.contains(name) ) return - if( name == NextflowDSLImpl.PROCESS_WHEN ) + if( name == DslCodeVisitor.PROCESS_WHEN ) return - if( name == NextflowDSLImpl.PROCESS_STUB ) + if( name == DslCodeVisitor.PROCESS_STUB ) return String message = "Unknown process directive: `$name`" diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ScriptParser.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ScriptParser.groovy index e491124003..28af74c85e 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/ScriptParser.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/ScriptParser.groovy @@ -23,9 +23,7 @@ import groovy.transform.CompileStatic import nextflow.Channel import nextflow.Nextflow import nextflow.Session -import nextflow.ast.NextflowDSL import nextflow.ast.NextflowXform -import nextflow.ast.OpXform import nextflow.exception.ScriptCompilationException import nextflow.extension.FilesEx import nextflow.file.FileHelper @@ -121,9 +119,7 @@ class ScriptParser { config = new CompilerConfiguration() config.addCompilationCustomizers( importCustomizer ) config.scriptBaseClass = BaseScript.class.name - config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowDSL)) config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowXform)) - config.addCompilationCustomizers( new ASTTransformationCustomizer(OpXform)) if( session?.debug ) config.debug = true diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ScriptTokens.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ScriptTokens.groovy index c7d38a1b7c..541f9d455f 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/ScriptTokens.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/ScriptTokens.groovy @@ -78,7 +78,7 @@ class TokenPathCall { * map( stdin, .. ) from x * * - * @see nextflow.ast.NextflowDSLImpl + * @see nextflow.ast.DslCodeVisitor * @see nextflow.script.params.TupleInParam#bind(java.lang.Object[]) */ class TokenStdinCall { } @@ -90,7 +90,7 @@ class TokenStdinCall { } * map( stdout, .. ) into x * * - * @see nextflow.ast.NextflowDSLImpl + * @see nextflow.ast.DslCodeVisitor * @see nextflow.script.params.TupleOutParam#bind(java.lang.Object[]) */ class TokenStdoutCall { } From 4612de33d166d5f8b44902bfb0149dd4148cbc28 Mon Sep 17 00:00:00 2001 From: Ben Sherman Date: Mon, 20 Nov 2023 05:25:40 -0600 Subject: [PATCH 02/36] Move process and workflow DSLs into separate classes Signed-off-by: Ben Sherman --- .../src/main/groovy/nextflow/Session.groovy | 4 +- .../groovy/nextflow/script/BaseScript.groovy | 49 +- .../nextflow/script/ProcessConfig.groovy | 779 +----------------- .../groovy/nextflow/script/ProcessDef.groovy | 39 +- .../nextflow/script/ProcessFactory.groovy | 16 +- .../groovy/nextflow/script/WorkflowDef.groovy | 60 +- .../nextflow/script/dsl/ProcessDsl.groovy | 739 +++++++++++++++++ .../nextflow/script/dsl/WorkflowDsl.groovy | 73 ++ .../groovy/nextflow/util/LoggerHelper.groovy | 3 +- ...lTest.groovy => DslCodeVisitorTest.groovy} | 4 +- ...rmTest.groovy => OperatorXformTest.groovy} | 6 +- .../nextflow/processor/TaskConfigTest.groovy | 123 ++- .../processor/TaskProcessorTest.groovy | 4 +- .../nextflow/scm/ProviderConfigTest.groovy | 28 + .../nextflow/script/IncludeDefTest.groovy | 4 +- .../nextflow/script/ProcessConfigTest.groovy | 643 --------------- .../nextflow/script/ProcessDefTest.groovy | 20 +- .../nextflow/script/ScriptMetaTest.groovy | 14 +- .../nextflow/script/WorkflowDefTest.groovy | 16 +- .../nextflow/script/dsl/ProcessDslTest.groovy | 640 ++++++++++++++ .../script/params/ParamsDsl2Test.groovy | 6 +- .../script/params/ParamsInTest.groovy | 24 +- 22 files changed, 1651 insertions(+), 1643 deletions(-) create mode 100644 modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessDsl.groovy create mode 100644 modules/nextflow/src/main/groovy/nextflow/script/dsl/WorkflowDsl.groovy rename modules/nextflow/src/test/groovy/nextflow/ast/{NextflowDSLImplTest.groovy => DslCodeVisitorTest.groovy} (98%) rename modules/nextflow/src/test/groovy/nextflow/ast/{OpXformTest.groovy => OperatorXformTest.groovy} (98%) create mode 100644 modules/nextflow/src/test/groovy/nextflow/script/dsl/ProcessDslTest.groovy diff --git a/modules/nextflow/src/main/groovy/nextflow/Session.groovy b/modules/nextflow/src/main/groovy/nextflow/Session.groovy index 668b1166ad..141c97f789 100644 --- a/modules/nextflow/src/main/groovy/nextflow/Session.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/Session.groovy @@ -55,6 +55,7 @@ import nextflow.processor.ErrorStrategy import nextflow.processor.TaskFault import nextflow.processor.TaskHandler import nextflow.processor.TaskProcessor +import nextflow.script.dsl.ProcessDsl import nextflow.script.BaseScript import nextflow.script.ProcessConfig import nextflow.script.ProcessFactory @@ -931,7 +932,7 @@ class Session implements ISession { * @return {@code true} if the name specified belongs to the list of process names or {@code false} otherwise */ protected boolean checkValidProcessName(Collection processNames, String selector, List errorMessage) { - final matches = processNames.any { name -> ProcessConfig.matchesSelector(name, selector) } + final matches = processNames.any { name -> ProcessDsl.matchesSelector(name, selector) } if( matches ) return true @@ -942,6 +943,7 @@ class Session implements ISession { errorMessage << message.toString() return false } + /** * Register a shutdown hook to close services when the session terminates * @param Closure diff --git a/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy b/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy index 58498fec71..e70a01c49c 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy @@ -23,6 +23,8 @@ import groovy.util.logging.Slf4j import nextflow.NextflowMeta import nextflow.Session import nextflow.exception.AbortOperationException +import nextflow.script.dsl.ProcessDsl +import nextflow.script.dsl.WorkflowDsl /** * Any user defined script will extends this class, it provides the base execution context * @@ -88,28 +90,51 @@ abstract class BaseScript extends Script implements ExecutionContext { binding.setVariable('moduleDir', meta.moduleDir ) } - protected process( String name, Closure body ) { - final process = new ProcessDef(this,body,name) + /** + * Define a process. + * + * @param name + * @param rawBody + */ + protected void process(String name, Closure rawBody) { + final builder = new ProcessDsl(this, name) + final copy = (Closure)rawBody.clone() + copy.delegate = builder + copy.resolveStrategy = Closure.DELEGATE_FIRST + final taskBody = copy.call() + final process = builder.withBody(taskBody).build() meta.addDefinition(process) } /** - * Workflow main entry point + * Define an anonymous workflow. * - * @param body The implementation body of the workflow - * @return The result of workflow execution + * @param rawBody */ - protected workflow(Closure workflowBody) { - // launch the execution - final workflow = new WorkflowDef(this, workflowBody) - // capture the main (unnamed) workflow definition + protected void workflow(Closure rawBody) { + final builder = new WorkflowDsl(this) + final copy = (Closure)rawBody.clone() + copy.delegate = builder + copy.resolveStrategy = Closure.DELEGATE_FIRST + final body = copy.call() + final workflow = builder.withBody(body).build() this.entryFlow = workflow - // add it to the list of workflow definitions meta.addDefinition(workflow) } - protected workflow(String name, Closure workflowDef) { - final workflow = new WorkflowDef(this,workflowDef,name) + /** + * Define a named workflow. + * + * @param name + * @param rawBody + */ + protected void workflow(String name, Closure rawBody) { + final builder = new WorkflowDsl(this, name) + final copy = (Closure)rawBody.clone() + copy.delegate = builder + copy.resolveStrategy = Closure.DELEGATE_FIRST + final body = copy.call() + final workflow = builder.withBody(body).build() meta.addDefinition(workflow) } diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessConfig.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessConfig.groovy index 5848b9aa38..fcd0b132fa 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessConfig.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessConfig.groovy @@ -16,21 +16,17 @@ package nextflow.script -import java.util.regex.Pattern - import groovy.transform.PackageScope import groovy.util.logging.Slf4j import nextflow.Const -import nextflow.NF -import nextflow.ast.DslCodeVisitor -import nextflow.exception.ConfigParseException -import nextflow.exception.IllegalDirectiveException import nextflow.executor.BashWrapperBuilder -import nextflow.processor.ConfigList import nextflow.processor.ErrorStrategy import nextflow.processor.TaskConfig import static nextflow.util.CacheHelper.HashMode -import nextflow.script.params.* +import nextflow.script.params.DefaultInParam +import nextflow.script.params.DefaultOutParam +import nextflow.script.params.InputsList +import nextflow.script.params.OutputsList /** * Holds the process configuration properties @@ -40,64 +36,6 @@ import nextflow.script.params.* @Slf4j class ProcessConfig implements Map, Cloneable { - static final public transient LABEL_REGEXP = ~/[a-zA-Z]([a-zA-Z0-9_]*[a-zA-Z0-9]+)?/ - - static final public List DIRECTIVES = [ - 'accelerator', - 'afterScript', - 'arch', - 'beforeScript', - 'cache', - 'conda', - 'cpus', - 'container', - 'containerOptions', - 'cleanup', - 'clusterOptions', - 'debug', - 'disk', - 'echo', // deprecated - 'errorStrategy', - 'executor', - 'ext', - 'fair', - 'machineType', - 'queue', - 'label', - 'maxSubmitAwait', - 'maxErrors', - 'maxForks', - 'maxRetries', - 'memory', - 'module', - 'penv', - 'pod', - 'publishDir', - 'scratch', - 'shell', - 'spack', - 'storeDir', - 'tag', - 'time', - // input-output qualifiers - 'file', - 'val', - 'each', - 'env', - 'secret', - 'stdin', - 'stdout', - 'stageInMode', - 'stageOutMode', - 'resourceLabels' - ] - - /** - * Names of directives that can be used more than once in the process definition - */ - @PackageScope - static final List repeatableDirectives = ['label','module','pod','publishDir'] - /** * Default directives values */ @@ -128,27 +66,16 @@ class ProcessConfig implements Map, Cloneable { */ private String processName - /** - * When {@code true} a {@link MissingPropertyException} is thrown when - * trying to access a property not existing - */ - private boolean throwExceptionOnMissingProperty - /** * List of process input definitions */ - private inputs = new InputsList() + private InputsList inputs = new InputsList() /** * List of process output definitions */ - private outputs = new OutputsList() + private OutputsList outputs = new OutputsList() - /** - * Initialize the taskConfig object with the defaults values - * - * @param script The owner {@code BaseScript} configuration object - */ protected ProcessConfig( BaseScript script ) { ownerScript = script configProperties = new LinkedHashMap() @@ -187,71 +114,6 @@ class ProcessConfig implements Map, Cloneable { return this } - /** - * Enable special behavior to allow the configuration object - * invoking directive method from the process DSL - * - * @param value {@code true} enable capture mode, {@code false} otherwise - * @return The object itself - */ - @PackageScope - ProcessConfig throwExceptionOnMissingProperty( boolean value ) { - this.throwExceptionOnMissingProperty = value - return this - } - - private void checkName(String name) { - if( DIRECTIVES.contains(name) ) - return - if( name == DslCodeVisitor.PROCESS_WHEN ) - return - if( name == DslCodeVisitor.PROCESS_STUB ) - return - - String message = "Unknown process directive: `$name`" - def alternatives = DIRECTIVES.closest(name) - if( alternatives.size()==1 ) { - message += '\n\nDid you mean of these?' - alternatives.each { - message += "\n $it" - } - } - throw new IllegalDirectiveException(message) - } - - Object invokeMethod(String name, Object args) { - /* - * This is need to patch #497 -- what is happening is that when in the config file - * is defined a directive like `memory`, `cpus`, etc in by using a closure, - * this closure is interpreted as method definition and it get invoked if a - * directive with the same name is defined in the process definition. - * To avoid that the offending property is removed from the map before the method - * is evaluated. - */ - if( configProperties.get(name) instanceof Closure ) - configProperties.remove(name) - - this.metaClass.invokeMethod(this,name,args) - } - - def methodMissing( String name, def args ) { - checkName(name) - - if( args instanceof Object[] ) { - if( args.size()==1 ) { - configProperties[ name ] = args[0] - } - else { - configProperties[ name ] = args.toList() - } - } - else { - configProperties[ name ] = args - } - - return this - } - @Override Object getProperty( String name ) { @@ -274,354 +136,30 @@ class ProcessConfig implements Map, Cloneable { default: if( configProperties.containsKey(name) ) return configProperties.get(name) - else if( throwExceptionOnMissingProperty ) - throw new MissingPropertyException("Unknown variable '$name'", name, null) else return null } } - Object put( String name, Object value ) { - - if( name in repeatableDirectives ) { - final result = configProperties.get(name) - configProperties.remove(name) - this.metaClass.invokeMethod(this, name, value) - return result - } - else { - return configProperties.put(name,value) - } - } - @PackageScope BaseScript getOwnerScript() { ownerScript } + @PackageScope + String getProcessName() { processName } + TaskConfig createTaskConfig() { return new TaskConfig(configProperties) } - /** - * Apply the settings defined in the configuration file for the given annotation label, for example: - * - * ``` - * process { - * withLabel: foo { - * cpus = 1 - * memory = 2.gb - * } - * } - * ``` - * - * @param configDirectives - * A map object modelling the setting defined defined by the user in the nextflow configuration file - * @param labels - * All the labels representing the object holding the configuration setting to apply - */ - protected void applyConfigSelectorWithLabels(Map configDirectives, List labels ) { - final prefix = 'withLabel:' - for( String rule : configDirectives.keySet() ) { - if( !rule.startsWith(prefix) ) - continue - final pattern = rule.substring(prefix.size()).trim() - if( !matchesLabels(labels, pattern) ) - continue - - log.debug "Config settings `$rule` matches labels `${labels.join(',')}` for process with name $processName" - def settings = configDirectives.get(rule) - if( settings instanceof Map ) { - applyConfigSettings(settings) - } - else if( settings != null ) { - throw new ConfigParseException("Unknown config settings for process labeled ${labels.join(',')} -- settings=$settings ") - } - } - } - - static boolean matchesLabels( List labels, String pattern ) { - final isNegated = pattern.startsWith('!') - if( isNegated ) - pattern = pattern.substring(1).trim() - - final regex = Pattern.compile(pattern) - for (label in labels) { - if (regex.matcher(label).matches()) { - return !isNegated - } - } - - return isNegated - } - - protected void applyConfigSelectorWithName(Map configDirectives, String target ) { - final prefix = 'withName:' - for( String rule : configDirectives.keySet() ) { - if( !rule.startsWith(prefix) ) - continue - final pattern = rule.substring(prefix.size()).trim() - if( !matchesSelector(target, pattern) ) - continue - - log.debug "Config settings `$rule` matches process $processName" - def settings = configDirectives.get(rule) - if( settings instanceof Map ) { - applyConfigSettings(settings) - } - else if( settings != null ) { - throw new ConfigParseException("Unknown config settings for process with name: $target -- settings=$settings ") - } - } - } - - static boolean matchesSelector( String name, String pattern ) { - final isNegated = pattern.startsWith('!') - if( isNegated ) - pattern = pattern.substring(1).trim() - return Pattern.compile(pattern).matcher(name).matches() ^ isNegated - } - - /** - * Apply the process configuration provided in the nextflow configuration file - * to the process instance - * - * @param configProcessScope The process configuration settings specified - * in the configuration file as {@link Map} object - * @param simpleName The process name - */ - void applyConfig(Map configProcessScope, String baseName, String simpleName, String fullyQualifiedName) { - // -- Apply the directives defined in the config object using the`withLabel:` syntax - final processLabels = this.getLabels() ?: [''] - this.applyConfigSelectorWithLabels(configProcessScope, processLabels) - - // -- apply setting defined in the config file using the process base name - this.applyConfigSelectorWithName(configProcessScope, baseName) - - // -- apply setting defined in the config file using the process simple name - if( simpleName && simpleName!=baseName ) - this.applyConfigSelectorWithName(configProcessScope, simpleName) - - // -- apply setting defined in the config file using the process qualified name (ie. with the execution scope) - if( fullyQualifiedName && (fullyQualifiedName!=simpleName || fullyQualifiedName!=baseName) ) - this.applyConfigSelectorWithName(configProcessScope, fullyQualifiedName) - - // -- Apply defaults - this.applyConfigDefaults(configProcessScope) - - // -- check for conflicting settings - if( this.scratch && this.stageInMode == 'rellink' ) { - log.warn("Directives `scratch` and `stageInMode=rellink` conflict with each other -- Enforcing default stageInMode for process `$simpleName`") - this.remove('stageInMode') - } - } - - void applyConfigLegacy(Map configProcessScope, String processName) { - applyConfig(configProcessScope, processName, null, null) - } - - - /** - * Apply the settings defined in the configuration file to the actual process configuration object - * - * @param settings - * A map object modelling the setting defined defined by the user in the nextflow configuration file - */ - protected void applyConfigSettings(Map settings) { - if( !settings ) - return - - for( Entry entry: settings ) { - if( entry.key.startsWith("withLabel:") || entry.key.startsWith("withName:")) - continue - - if( !DIRECTIVES.contains(entry.key) ) - log.warn "Unknown directive `$entry.key` for process `$processName`" - - if( entry.key == 'params' ) // <-- patch issue #242 - continue - - if( entry.key == 'ext' ) { - if( this.getProperty('ext') instanceof Map ) { - // update missing 'ext' properties found in 'process' scope - def ext = this.getProperty('ext') as Map - entry.value.each { String k, v -> ext[k] = v } - } - continue - } - - this.put(entry.key,entry.value) - } - } - - /** - * Apply the process settings defined globally in the process config scope - * - * @param processDefaults - * A map object representing the setting to be applied to the process - * (provided it does not already define a different value for - * the same config setting). - * - */ - protected void applyConfigDefaults( Map processDefaults ) { - for( String key : processDefaults.keySet() ) { - if( key == 'params' ) - continue - final value = processDefaults.get(key) - final current = this.getProperty(key) - if( key == 'ext' ) { - if( value instanceof Map && current instanceof Map ) { - final ext = current as Map - value.each { k,v -> if(!ext.containsKey(k)) ext.put(k,v) } - } - } - else if( !this.containsKey(key) || (DEFAULT_CONFIG.containsKey(key) && current==DEFAULT_CONFIG.get(key)) ) { - this.put(key, value) - } - } - } - - /** - * Type shortcut to {@code #configProperties.inputs} - */ InputsList getInputs() { inputs } - /** - * Type shortcut to {@code #configProperties.outputs} - */ OutputsList getOutputs() { outputs } - /** - * Implements the process {@code debug} directive. - */ - ProcessConfig debug( value ) { - configProperties.debug = value - return this - } - - /** - * Implements the process {@code echo} directive for backwards compatibility. - * - * note: without this method definition {@link BaseScript#echo} will be invoked - */ - ProcessConfig echo( value ) { - log.warn1('The `echo` directive has been deprecated - use to `debug` instead') - configProperties.debug = value - return this - } - - /// input parameters - - InParam _in_val( obj ) { - new ValueInParam(this).bind(obj) - } - - InParam _in_file( obj ) { - new FileInParam(this).bind(obj) - } - - InParam _in_path( Map opts=null, obj ) { - new FileInParam(this) - .setPathQualifier(true) - .setOptions(opts) - .bind(obj) - } - - InParam _in_each( obj ) { - new EachInParam(this).bind(obj) - } - - InParam _in_tuple( Object... obj ) { - new TupleInParam(this).bind(obj) - } - - InParam _in_stdin( obj = null ) { - def result = new StdInParam(this) - if( obj ) result.bind(obj) - result - } - - InParam _in_env( obj ) { - new EnvInParam(this).bind(obj) - } - - - /// output parameters - - OutParam _out_val( Object obj ) { - new ValueOutParam(this).bind(obj) - } - - OutParam _out_val( Map opts, Object obj ) { - new ValueOutParam(this) - .setOptions(opts) - .bind(obj) - } - - OutParam _out_env( Object obj ) { - new EnvOutParam(this).bind(obj) - } - - OutParam _out_env( Map opts, Object obj ) { - new EnvOutParam(this) - .setOptions(opts) - .bind(obj) - } - - - OutParam _out_file( Object obj ) { - // note: check that is a String type to avoid to force - // the evaluation of GString object to a string - if( obj instanceof String && obj == '-' ) - new StdOutParam(this).bind(obj) - - else - new FileOutParam(this).bind(obj) - } - - OutParam _out_path( Map opts=null, Object obj ) { - // note: check that is a String type to avoid to force - // the evaluation of GString object to a string - if( obj instanceof String && obj == '-' ) { - new StdOutParam(this) - .setOptions(opts) - .bind(obj) - } - else { - new FileOutParam(this) - .setPathQualifier(true) - .setOptions(opts) - .bind(obj) - } - } - - OutParam _out_tuple( Object... obj ) { - new TupleOutParam(this) .bind(obj) - } - - OutParam _out_tuple( Map opts, Object... obj ) { - new TupleOutParam(this) - .setOptions(opts) - .bind(obj) - } - - OutParam _out_stdout( Map opts ) { - new StdOutParam(this) - .setOptions(opts) - .bind('-') - } - - OutParam _out_stdout( obj = null ) { - def result = new StdOutParam(this).bind('-') - if( obj ) { - result.into(obj) - } - result - } - /** * Defines a special *dummy* input parameter, when no inputs are * provided by the user for the current task @@ -652,73 +190,6 @@ class ProcessConfig implements Map, Cloneable { HashMode.of(configProperties.cache) ?: HashMode.DEFAULT() } - protected boolean isValidLabel(String lbl) { - def p = lbl.indexOf('=') - if( p==-1 ) - return LABEL_REGEXP.matcher(lbl).matches() - - def left = lbl.substring(0,p) - def right = lbl.substring(p+1) - return LABEL_REGEXP.matcher(left).matches() && LABEL_REGEXP.matcher(right).matches() - } - - /** - * Implements the process {@code label} directive. - * - * Note this directive can be specified (invoked) more than one time in - * the process context. - * - * @param lbl - * The label to be attached to the process. - * @return - * The {@link ProcessConfig} instance itself. - */ - ProcessConfig label(String lbl) { - if( !lbl ) return this - - // -- check that label has a valid syntax - if( !isValidLabel(lbl) ) - throw new IllegalConfigException("Not a valid process label: $lbl -- Label must consist of alphanumeric characters or '_', must start with an alphabetic character and must end with an alphanumeric character") - - // -- get the current label, it must be a list - def allLabels = (List)configProperties.get('label') - if( !allLabels ) { - allLabels = new ConfigList() - configProperties.put('label', allLabels) - } - - // -- avoid duplicates - if( !allLabels.contains(lbl) ) - allLabels.add(lbl) - return this - } - - /** - * Implements the process {@code label} directive. - * - * Note this directive can be specified (invoked) more than one time in - * the process context. - * - * @param map - * The map to be attached to the process. - * @return - * The {@link ProcessConfig} instance itself. - */ - ProcessConfig resourceLabels(Map map) { - if( !map ) - return this - - // -- get the current sticker, it must be a Map - def allLabels = (Map)configProperties.get('resourceLabels') - if( !allLabels ) { - allLabels = [:] - } - // -- merge duplicates - allLabels += map - configProperties.put('resourceLabels', allLabels) - return this - } - Map getResourceLabels() { (configProperties.get('resourceLabels') ?: Collections.emptyMap()) as Map } @@ -740,240 +211,8 @@ class ProcessConfig implements Map, Cloneable { throw new IllegalArgumentException("Unexpected value for directive `fair` -- offending value: $value") } - ProcessConfig secret(String name) { - if( !name ) - return this - - // -- get the current label, it must be a list - def allSecrets = (List)configProperties.get('secret') - if( !allSecrets ) { - allSecrets = new ConfigList() - configProperties.put('secret', allSecrets) - } - - // -- avoid duplicates - if( !allSecrets.contains(name) ) - allSecrets.add(name) - return this - } - List getSecret() { (List) configProperties.get('secret') ?: Collections.emptyList() } - /** - * Implements the process {@code module} directive. - * - * See also http://modules.sourceforge.net - * - * @param moduleName - * The module name to be used to execute the process. - * @return - * The {@link ProcessConfig} instance itself. - */ - ProcessConfig module( moduleName ) { - // when no name is provided, just exit - if( !moduleName ) - return this - - def result = (List)configProperties.module - if( result == null ) { - result = new ConfigList() - configProperties.put('module', result) - } - - if( moduleName instanceof List ) - result.addAll(moduleName) - else - result.add(moduleName) - return this - } - - /** - * Implements the {@code errorStrategy} directive - * - * @see ErrorStrategy - * - * @param strategy - * A string representing the error strategy to be used. - * @return - * The {@link ProcessConfig} instance itself. - */ - ProcessConfig errorStrategy( strategy ) { - if( strategy instanceof CharSequence && !ErrorStrategy.isValid(strategy) ) { - throw new IllegalArgumentException("Unknown error strategy '${strategy}' ― Available strategies are: ${ErrorStrategy.values().join(',').toLowerCase()}") - } - - configProperties.put('errorStrategy', strategy) - return this - } - - /** - * Allow the user to specify publishDir directive as a map eg: - * - * publishDir path:'/some/dir', mode: 'copy' - * - * @param params - * A map representing the the publishDir setting - * @return - * The {@link ProcessConfig} instance itself - */ - ProcessConfig publishDir(Map params) { - if( !params ) - return this - - def dirs = (List)configProperties.get('publishDir') - if( !dirs ) { - dirs = new ConfigList() - configProperties.put('publishDir', dirs) - } - - dirs.add(params) - return this - } - - /** - * Allow the user to specify publishDir directive with a path and a list of named parameters, eg: - * - * publishDir '/some/dir', mode: 'copy' - * - * @param params - * A map representing the publishDir properties - * @param target - * The target publishDir path - * @return - * The {@link ProcessConfig} instance itself - */ - ProcessConfig publishDir(Map params, target) { - params.put('path', target) - publishDir( params ) - } - - /** - * Allow the user to specify the publishDir as a string path, eg: - * - * publishDir '/some/dir' - * - * @param target - * The target publishDir path - * @return - * The {@link ProcessConfig} instance itself - */ - ProcessConfig publishDir( target ) { - if( target instanceof List ) { - for( Object item : target ) { publishDir(item) } - } - else if( target instanceof Map ) { - publishDir( target as Map ) - } - else { - publishDir([path: target]) - } - return this - } - - /** - * Allow use to specify K8s `pod` options - * - * @param entry - * A map object representing pod config options - * @return - * The {@link ProcessConfig} instance itself - */ - ProcessConfig pod( Map entry ) { - - if( !entry ) - return this - - def allOptions = (List)configProperties.get('pod') - if( !allOptions ) { - allOptions = new ConfigList() - configProperties.put('pod', allOptions) - } - - allOptions.add(entry) - return this - - } - - ProcessConfig accelerator( Map params, value ) { - if( value instanceof Number ) { - if( params.limit==null ) - params.limit=value - else if( params.request==null ) - params.request=value - } - else if( value != null ) - throw new IllegalArgumentException("Not a valid `accelerator` directive value: $value [${value.getClass().getName()}]") - accelerator(params) - return this - } - - ProcessConfig accelerator( value ) { - if( value instanceof Number ) - configProperties.put('accelerator', [limit: value]) - else if( value instanceof Map ) - configProperties.put('accelerator', value) - else if( value != null ) - throw new IllegalArgumentException("Not a valid `accelerator` directive value: $value [${value.getClass().getName()}]") - return this - } - - /** - * Allow user to specify `disk` directive as a value with a list of options, eg: - * - * disk 375.GB, type: 'local-ssd' - * - * @param opts - * A map representing the disk options - * @param value - * The default disk value - * @return - * The {@link ProcessConfig} instance itself - */ - ProcessConfig disk( Map opts, value ) { - opts.request = value - return disk(opts) - } - - /** - * Allow user to specify `disk` directive as a value or a list of options, eg: - * - * disk 100.GB - * disk request: 375.GB, type: 'local-ssd' - * - * @param value - * The default disk value or map of options - * @return - * The {@link ProcessConfig} instance itself - */ - ProcessConfig disk( value ) { - if( value instanceof Map || value instanceof Closure ) - configProperties.put('disk', value) - else - configProperties.put('disk', [request: value]) - return this - } - - ProcessConfig arch( Map params, value ) { - if( value instanceof String ) { - if( params.name==null ) - params.name=value - } - else if( value != null ) - throw new IllegalArgumentException("Not a valid `arch` directive value: $value [${value.getClass().getName()}]") - arch(params) - return this - } - - ProcessConfig arch( value ) { - if( value instanceof String ) - configProperties.put('arch', [name: value]) - else if( value instanceof Map ) - configProperties.put('arch', value) - else if( value != null ) - throw new IllegalArgumentException("Not a valid `arch` directive value: $value [${value.getClass().getName()}]") - return this - } - } diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy index 7cfc6309cc..b308641415 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy @@ -23,6 +23,7 @@ import nextflow.Global import nextflow.Session import nextflow.exception.ScriptRuntimeException import nextflow.extension.CH +import nextflow.script.dsl.ProcessDsl import nextflow.script.params.BaseInParam import nextflow.script.params.BaseOutParam import nextflow.script.params.EachInParam @@ -62,32 +63,28 @@ class ProcessDef extends BindableDef implements IterableDef, ChainableDef { */ private String baseName - /** - * The closure holding the process definition body - */ - private Closure rawBody - /** * The resolved process configuration */ - private transient ProcessConfig processConfig + private ProcessConfig processConfig /** * The actual process implementation */ - private transient BodyDef taskBody + private BodyDef taskBody /** * The result of the process execution */ private transient ChannelOut output - ProcessDef(BaseScript owner, Closure body, String name ) { + ProcessDef(BaseScript owner, String name, BodyDef body, ProcessConfig config) { this.owner = owner - this.rawBody = body this.simpleName = name this.processName = name this.baseName = name + this.taskBody = body + this.processConfig = config } static String stripScope(String str) { @@ -95,32 +92,15 @@ class ProcessDef extends BindableDef implements IterableDef, ChainableDef { } protected void initialize() { - log.trace "Process config > $processName" - assert processConfig==null - - // the config object - processConfig = new ProcessConfig(owner,processName) - - // Invoke the code block which will return the script closure to the executed. - // As side effect will set all the property declarations in the 'taskConfig' object. - processConfig.throwExceptionOnMissingProperty(true) - final copy = (Closure)rawBody.clone() - copy.setResolveStrategy(Closure.DELEGATE_FIRST) - copy.setDelegate(processConfig) - taskBody = copy.call() as BodyDef - processConfig.throwExceptionOnMissingProperty(false) - if ( !taskBody ) - throw new ScriptRuntimeException("Missing script in the specified process block -- make sure it terminates with the script string to be executed") - // apply config settings to the process - processConfig.applyConfig((Map)session.config.process, baseName, simpleName, processName) + new ProcessDsl(processConfig).applyConfig((Map)session.config.process, baseName, simpleName, processName) } @Override ProcessDef clone() { def result = (ProcessDef)super.clone() - result.@taskBody = taskBody?.clone() - result.@rawBody = (Closure)rawBody?.clone() + result.@taskBody = taskBody.clone() + result.@processConfig = processConfig.clone() return result } @@ -130,6 +110,7 @@ class ProcessDef extends BindableDef implements IterableDef, ChainableDef { def result = clone() result.@processName = name result.@simpleName = stripScope(name) + result.@processConfig.processName = name return result } diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessFactory.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessFactory.groovy index 56dd23838a..3d660c8774 100755 --- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessFactory.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessFactory.groovy @@ -23,6 +23,7 @@ import nextflow.Session import nextflow.executor.Executor import nextflow.executor.ExecutorFactory import nextflow.processor.TaskProcessor +import nextflow.script.dsl.ProcessDsl /** * Factory class for {@TaskProcessor} instances * @@ -79,25 +80,26 @@ class ProcessFactory { * @return * The {@code Processor} instance */ + @Deprecated TaskProcessor createProcessor( String name, Closure body ) { assert body assert config.process instanceof Map - // -- the config object - final processConfig = new ProcessConfig(owner, name) + final builder = new ProcessDsl(owner, name) // Invoke the code block which will return the script closure to the executed. // As side effect will set all the property declarations in the 'taskConfig' object. - processConfig.throwExceptionOnMissingProperty(true) final copy = (Closure)body.clone() - copy.setResolveStrategy(Closure.DELEGATE_FIRST) - copy.setDelegate(processConfig) + copy.delegate = builder + copy.resolveStrategy = Closure.DELEGATE_FIRST final script = copy.call() - processConfig.throwExceptionOnMissingProperty(false) if ( !script ) throw new IllegalArgumentException("Missing script in the specified process block -- make sure it terminates with the script string to be executed") // -- apply settings from config file to process config - processConfig.applyConfigLegacy((Map)config.process, name) + builder.applyConfig((Map)config.process, name, null, null) + + // -- the config object + final processConfig = builder.getConfig() // -- get the executor for the given process config final execObj = executorFactory.getExecutor(name, processConfig, script, session) diff --git a/modules/nextflow/src/main/groovy/nextflow/script/WorkflowDef.groovy b/modules/nextflow/src/main/groovy/nextflow/script/WorkflowDef.groovy index ec54d3043a..3b5d4b5cf8 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/WorkflowDef.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/WorkflowDef.groovy @@ -52,18 +52,12 @@ class WorkflowDef extends BindableDef implements ChainableDef, IterableDef, Exec private WorkflowBinding binding - WorkflowDef(BaseScript owner, Closure rawBody, String name=null) { + WorkflowDef(BaseScript owner, String name, BodyDef body, List takes, List emits) { this.owner = owner this.name = name - // invoke the body resolving in/out params - final copy = (Closure)rawBody.clone() - final resolver = new WorkflowParamsResolver() - copy.setResolveStrategy(Closure.DELEGATE_FIRST) - copy.setDelegate(resolver) - this.body = copy.call() - // now it can access the parameters - this.declaredInputs = new ArrayList<>(resolver.getTakes().keySet()) - this.declaredOutputs = new ArrayList<>(resolver.getEmits().keySet()) + this.body = body + this.declaredInputs = takes + this.declaredOutputs = emits this.variableNames = getVarNames0() } @@ -208,49 +202,3 @@ class WorkflowDef extends BindableDef implements ChainableDef, IterableDef, Exec } } - -/** - * Hold workflow parameters - */ -@Slf4j -@CompileStatic -class WorkflowParamsResolver { - - static final private String TAKE_PREFIX = '_take_' - static final private String EMIT_PREFIX = '_emit_' - - - Map takes = new LinkedHashMap<>(10) - Map emits = new LinkedHashMap<>(10) - - @Override - def invokeMethod(String name, Object args) { - if( name.startsWith(TAKE_PREFIX) ) - takes.put(name.substring(TAKE_PREFIX.size()), args) - - else if( name.startsWith(EMIT_PREFIX) ) - emits.put(name.substring(EMIT_PREFIX.size()), args) - - else - throw new MissingMethodException(name, WorkflowDef, args) - } - - private Map argsToMap(Object args) { - if( args && args.getClass().isArray() ) { - if( ((Object[])args)[0] instanceof Map ) { - def map = (Map)((Object[])args)[0] - return new HashMap(map) - } - } - Collections.emptyMap() - } - - private Map argToPublishOpts(Object args) { - final opts = argsToMap(args) - if( opts.containsKey('saveAs')) { - log.warn "Workflow publish does not support `saveAs` option" - opts.remove('saveAs') - } - return opts - } -} diff --git a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessDsl.groovy b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessDsl.groovy new file mode 100644 index 0000000000..f4da53c4db --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessDsl.groovy @@ -0,0 +1,739 @@ +/* + * Copyright 2013-2023, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.script.dsl + +import java.util.regex.Pattern + +import groovy.util.logging.Slf4j +import nextflow.ast.DslCodeVisitor +import nextflow.exception.ConfigParseException +import nextflow.exception.IllegalConfigException +import nextflow.exception.IllegalDirectiveException +import nextflow.exception.ScriptRuntimeException +import nextflow.processor.ConfigList +import nextflow.processor.ErrorStrategy +import nextflow.script.params.* +import nextflow.script.BaseScript +import nextflow.script.BodyDef +import nextflow.script.ProcessConfig +import nextflow.script.ProcessDef + +/** + * Implements the process DSL. + * + * @author Ben Sherman + */ +@Slf4j +class ProcessDsl { + + static final List DIRECTIVES = [ + 'accelerator', + 'afterScript', + 'arch', + 'beforeScript', + 'cache', + 'cleanup', + 'clusterOptions', + 'conda', + 'container', + 'containerOptions', + 'cpus', + 'debug', + 'disk', + 'echo', // deprecated + 'errorStrategy', + 'executor', + 'ext', + 'fair', + 'label', + 'machineType', + 'maxErrors', + 'maxForks', + 'maxRetries', + 'maxSubmitAwait', + 'memory', + 'module', + 'penv', + 'pod', + 'publishDir', + 'queue', + 'resourceLabels', + 'scratch', + 'secret', + 'shell', + 'spack', + 'stageInMode', + 'stageOutMode', + 'storeDir', + 'tag', + 'time' + ] + + private BaseScript ownerScript + private String processName + private BodyDef body + private ProcessConfig config + + ProcessDsl(BaseScript ownerScript, String processName) { + this.ownerScript = ownerScript + this.processName = processName + this.config = new ProcessConfig(ownerScript, processName) + } + + ProcessDsl(ProcessConfig config) { + this.ownerScript = config.getOwnerScript() + this.processName = config.getProcessName() + this.config = config + } + + Object invokeMethod(String name, Object args) { + /* + * This is need to patch #497 -- what is happening is that when in the config file + * is defined a directive like `memory`, `cpus`, etc in by using a closure, + * this closure is interpreted as method definition and it get invoked if a + * directive with the same name is defined in the process definition. + * To avoid that the offending property is removed from the map before the method + * is evaluated. + */ + if( config.get(name) instanceof Closure ) + config.remove(name) + + this.metaClass.invokeMethod(this,name,args) + } + + def methodMissing( String name, def args ) { + checkName(name) + + if( args instanceof Object[] ) + config.put(name, args.size()==1 ? args[0] : args.toList()) + else + config.put(name, args) + } + + private void checkName(String name) { + if( DIRECTIVES.contains(name) ) + return + if( name == DslCodeVisitor.PROCESS_WHEN ) + return + if( name == DslCodeVisitor.PROCESS_STUB ) + return + + String message = "Unknown process directive: `$name`" + def alternatives = DIRECTIVES.closest(name) + if( alternatives.size()==1 ) { + message += '\n\nDid you mean one of these?' + alternatives.each { + message += "\n $it" + } + } + throw new IllegalDirectiveException(message) + } + + /// DIRECTIVES + + void accelerator( Map params, value ) { + if( value instanceof Number ) { + if( params.limit==null ) + params.limit=value + else if( params.request==null ) + params.request=value + } + else if( value != null ) + throw new IllegalArgumentException("Not a valid `accelerator` directive value: $value [${value.getClass().getName()}]") + accelerator(params) + } + + void accelerator( value ) { + if( value instanceof Number ) + config.put('accelerator', [limit: value]) + else if( value instanceof Map ) + config.put('accelerator', value) + else if( value != null ) + throw new IllegalArgumentException("Not a valid `accelerator` directive value: $value [${value.getClass().getName()}]") + } + + void arch( Map params, value ) { + if( value instanceof String ) { + if( params.name==null ) + params.name=value + } + else if( value != null ) + throw new IllegalArgumentException("Not a valid `arch` directive value: $value [${value.getClass().getName()}]") + arch(params) + } + + void arch( value ) { + if( value instanceof String ) + config.put('arch', [name: value]) + else if( value instanceof Map ) + config.put('arch', value) + else if( value != null ) + throw new IllegalArgumentException("Not a valid `arch` directive value: $value [${value.getClass().getName()}]") + } + + void debug(boolean value) { + config.debug = value + } + + /** + * Implements the {@code disk} directive, e.g.: + * + * disk 375.GB, type: 'local-ssd' + * + * @param opts + * @param value + */ + void disk( Map opts, value ) { + opts.request = value + disk(opts) + } + + /** + * Implements the {@code disk} directive, e.g.: + * + * disk 100.GB + * disk request: 375.GB, type: 'local-ssd' + * + * @param value + */ + void disk( value ) { + if( value instanceof Map || value instanceof Closure ) + config.put('disk', value) + else + config.put('disk', [request: value]) + } + + /** + * Implements the {@code echo} directive for backwards compatibility. + * + * note: without this method definition {@link BaseScript#echo} will be invoked + * + * @param value + */ + void echo( value ) { + log.warn1('The `echo` directive has been deprecated - use `debug` instead') + config.put('debug', value) + } + + /** + * Implements the {@code errorStrategy} directive. + * + * @param strategy + */ + void errorStrategy( CharSequence strategy ) { + if( !ErrorStrategy.isValid(strategy) ) + throw new IllegalArgumentException("Unknown error strategy '${strategy}' ― Available strategies are: ${ErrorStrategy.values().join(',').toLowerCase()}") + + config.put('errorStrategy', strategy) + } + + /** + * Implements the {@code label} directive. + * + * This directive can be specified (invoked) more than once in + * the process definition. + * + * @param lbl + */ + void label(String lbl) { + if( !lbl ) return + + // -- check that label has a valid syntax + if( !isValidLabel(lbl) ) + throw new IllegalConfigException("Not a valid process label: $lbl -- Label must consist of alphanumeric characters or '_', must start with an alphabetic character and must end with an alphanumeric character") + + // -- get the current label, it must be a list + def allLabels = (List)config.get('label') + if( !allLabels ) { + allLabels = new ConfigList() + config.put('label', allLabels) + } + + // -- avoid duplicates + if( !allLabels.contains(lbl) ) + allLabels.add(lbl) + } + + private static final Pattern LABEL_REGEXP = ~/[a-zA-Z]([a-zA-Z0-9_]*[a-zA-Z0-9]+)?/ + + protected static boolean isValidLabel(String lbl) { + def p = lbl.indexOf('=') + if( p==-1 ) + return LABEL_REGEXP.matcher(lbl).matches() + + def left = lbl.substring(0,p) + def right = lbl.substring(p+1) + return LABEL_REGEXP.matcher(left).matches() && LABEL_REGEXP.matcher(right).matches() + } + + /** + * Implements the {@code module} directive. + * + * See also http://modules.sourceforge.net + * + * @param value + */ + void module( String value ) { + if( !value ) return + + def result = (List)config.module + if( result == null ) { + result = new ConfigList() + config.put('module', result) + } + + result.add(value) + } + + /** + * Implements the {@code pod} directive. + * + * @param entry + */ + void pod( Map entry ) { + if( !entry ) return + + def allOptions = (List)config.get('pod') + if( !allOptions ) { + allOptions = new ConfigList() + config.put('pod', allOptions) + } + + allOptions.add(entry) + } + + /** + * Implements the {@code publishDir} directive as a map eg: + * + * publishDir path: '/some/dir', mode: 'copy' + * + * This directive can be specified (invoked) multiple times in + * the process definition. + * + * @param params + */ + void publishDir(Map params) { + if( !params ) return + + def dirs = (List)config.get('publishDir') + if( !dirs ) { + dirs = new ConfigList() + config.put('publishDir', dirs) + } + + dirs.add(params) + } + + /** + * Implements the {@code publishDir} directive as a path with named parameters, eg: + * + * publishDir '/some/dir', mode: 'copy' + * + * @param params + * @param path + */ + void publishDir(Map params, CharSequence path) { + params.put('path', path) + publishDir( params ) + } + + /** + * Implements the {@code publishDir} directive as a string path, eg: + * + * publishDir '/some/dir' + * + * @param target + */ + void publishDir( target ) { + if( target instanceof List ) { + for( Object item : target ) { publishDir(item) } + } + else if( target instanceof Map ) { + publishDir( target as Map ) + } + else { + publishDir([path: target]) + } + } + + /** + * Implements the {@code resourceLabels} directive. + * + * This directive can be specified (invoked) multiple times in + * the process definition. + * + * @param map + */ + void resourceLabels(Map map) { + if( !map ) return + + // -- get the current sticker, it must be a Map + def allLabels = (Map)config.get('resourceLabels') + if( !allLabels ) { + allLabels = [:] + } + // -- merge duplicates + allLabels += map + config.put('resourceLabels', allLabels) + } + + /** + * Implements the {@code secret} directive. + * + * This directive can be specified (invoked) multiple times in + * the process definition. + * + * @param name + */ + void secret(String name) { + if( !name ) return + + // -- get the current label, it must be a list + def allSecrets = (List)config.get('secret') + if( !allSecrets ) { + allSecrets = new ConfigList() + config.put('secret', allSecrets) + } + + // -- avoid duplicates + if( !allSecrets.contains(name) ) + allSecrets.add(name) + } + + /// INPUTS + + InParam _in_each( Object obj ) { + new EachInParam(config).bind(obj) + } + + InParam _in_env( Object obj ) { + new EnvInParam(config).bind(obj) + } + + InParam _in_file( Object obj ) { + new FileInParam(config).bind(obj) + } + + InParam _in_path( Map opts=null, Object obj ) { + new FileInParam(config) + .setPathQualifier(true) + .setOptions(opts) + .bind(obj) + } + + InParam _in_stdin( Object obj = null ) { + def result = new StdInParam(config) + if( obj ) + result.bind(obj) + result + } + + InParam _in_tuple( Object... obj ) { + new TupleInParam(config).bind(obj) + } + + InParam _in_val( Object obj ) { + new ValueInParam(config).bind(obj) + } + + /// OUTPUTS + + OutParam _out_env( Object obj ) { + new EnvOutParam(config) + .bind(obj) + } + + OutParam _out_env( Map opts, Object obj ) { + new EnvOutParam(config) + .setOptions(opts) + .bind(obj) + } + + OutParam _out_file( Object obj ) { + // note: check that is a String type to avoid to force + // the evaluation of GString object to a string + if( obj instanceof String && obj == '-' ) + new StdOutParam(config).bind(obj) + else + new FileOutParam(config).bind(obj) + } + + OutParam _out_path( Map opts=null, Object obj ) { + // note: check that is a String type to avoid to force + // the evaluation of GString object to a string + if( obj instanceof String && obj == '-' ) + new StdOutParam(config) + .setOptions(opts) + .bind(obj) + + else + new FileOutParam(config) + .setPathQualifier(true) + .setOptions(opts) + .bind(obj) + } + + OutParam _out_stdout( Map opts ) { + new StdOutParam(config) + .setOptions(opts) + .bind('-') + } + + OutParam _out_stdout( obj = null ) { + def result = new StdOutParam(config).bind('-') + if( obj ) + result.setInto(obj) + result + } + + OutParam _out_tuple( Object... obj ) { + new TupleOutParam(config) + .bind(obj) + } + + OutParam _out_tuple( Map opts, Object... obj ) { + new TupleOutParam(config) + .setOptions(opts) + .bind(obj) + } + + OutParam _out_val( Object obj ) { + new ValueOutParam(config) + .bind(obj) + } + + OutParam _out_val( Map opts, Object obj ) { + new ValueOutParam(config) + .setOptions(opts) + .bind(obj) + } + + /// SCRIPT + + ProcessDsl withBody(BodyDef body) { + this.body = body + return this + } + + /// CONFIG + + /** + * Apply process config settings from the config file to a process. + * + * @param configDirectives + * @param baseName + * @param simpleName + * @param fullyQualifiedName + */ + void applyConfig(Map configDirectives, String baseName, String simpleName, String fullyQualifiedName) { + // -- apply settings defined in the config object using the`withLabel:` syntax + final processLabels = config.getLabels() ?: [''] + applyConfigSelectorWithLabels(configDirectives, processLabels) + + // -- apply settings defined in the config file using the process base name + applyConfigSelectorWithName(configDirectives, baseName) + + // -- apply settings defined in the config file using the process simple name + if( simpleName && simpleName!=baseName ) + applyConfigSelectorWithName(configDirectives, simpleName) + + // -- apply settings defined in the config file using the process fully qualified name (ie. with the execution scope) + if( fullyQualifiedName && (fullyQualifiedName!=simpleName || fullyQualifiedName!=baseName) ) + applyConfigSelectorWithName(configDirectives, fullyQualifiedName) + + // -- apply defaults + applyConfigDefaults(configDirectives) + + // -- check for conflicting settings + if( config.scratch && config.stageInMode == 'rellink' ) { + log.warn("Directives `scratch` and `stageInMode=rellink` conflict with each other -- Enforcing default stageInMode for process `$simpleName`") + config.remove('stageInMode') + } + } + + /** + * Apply the config settings in a label selector, for example: + * + * ``` + * process { + * withLabel: foo { + * cpus = 1 + * memory = 2.gb + * } + * } + * ``` + * + * @param configDirectives + * @param labels + */ + protected void applyConfigSelectorWithLabels(Map configDirectives, List labels) { + final prefix = 'withLabel:' + for( String rule : configDirectives.keySet() ) { + if( !rule.startsWith(prefix) ) + continue + final pattern = rule.substring(prefix.size()).trim() + if( !matchesLabels(labels, pattern) ) + continue + + log.debug "Config settings `$rule` matches labels `${labels.join(',')}` for process with name $processName" + final settings = configDirectives.get(rule) + if( settings instanceof Map ) { + applyConfigSettings(settings) + } + else if( settings != null ) { + throw new ConfigParseException("Unknown config settings for process labeled ${labels.join(',')} -- settings=$settings ") + } + } + } + + static boolean matchesLabels(List labels, String pattern) { + final isNegated = pattern.startsWith('!') + if( isNegated ) + pattern = pattern.substring(1).trim() + + final regex = Pattern.compile(pattern) + for (label in labels) { + if (regex.matcher(label).matches()) { + return !isNegated + } + } + + return isNegated + } + + /** + * Apply the config settings in a name selector, for example: + * + * ``` + * process { + * withName: foo { + * cpus = 1 + * memory = 2.gb + * } + * } + * ``` + * + * @param configDirectives + * @param target + */ + protected void applyConfigSelectorWithName(Map configDirectives, String target) { + final prefix = 'withName:' + for( String rule : configDirectives.keySet() ) { + if( !rule.startsWith(prefix) ) + continue + final pattern = rule.substring(prefix.size()).trim() + if( !matchesSelector(target, pattern) ) + continue + + log.debug "Config settings `$rule` matches process $processName" + def settings = configDirectives.get(rule) + if( settings instanceof Map ) { + applyConfigSettings(settings) + } + else if( settings != null ) { + throw new ConfigParseException("Unknown config settings for process with name: $target -- settings=$settings ") + } + } + } + + static boolean matchesSelector(String name, String pattern) { + final isNegated = pattern.startsWith('!') + if( isNegated ) + pattern = pattern.substring(1).trim() + return Pattern.compile(pattern).matcher(name).matches() ^ isNegated + } + + + /** + * Apply config settings to a process. + * + * @param settings + */ + protected void applyConfigSettings(Map settings) { + if( !settings ) + return + + for( def entry : settings ) { + if( entry.key.startsWith("withLabel:") || entry.key.startsWith("withName:")) + continue + + if( !DIRECTIVES.contains(entry.key) ) + log.warn "Unknown directive `$entry.key` for process `$processName`" + + if( entry.key == 'params' ) // <-- patch issue #242 + continue + + if( entry.key == 'ext' ) { + if( config.getProperty('ext') instanceof Map ) { + // update missing 'ext' properties found in 'process' scope + def ext = config.getProperty('ext') as Map + entry.value.each { String k, v -> ext[k] = v } + } + continue + } + + putWithRepeat(entry.key, entry.value) + } + } + + /** + * Apply the global settings in the process config scope to a process. + * + * @param defaults + */ + protected void applyConfigDefaults( Map defaults ) { + for( String key : defaults.keySet() ) { + if( key == 'params' ) + continue + final value = defaults.get(key) + final current = config.getProperty(key) + if( key == 'ext' ) { + if( value instanceof Map && current instanceof Map ) { + final ext = current as Map + value.each { k,v -> if(!ext.containsKey(k)) ext.put(k,v) } + } + } + else if( !config.containsKey(key) || (ProcessConfig.DEFAULT_CONFIG.containsKey(key) && current==ProcessConfig.DEFAULT_CONFIG.get(key)) ) { + putWithRepeat(key, value) + } + } + } + + private static final List REPEATABLE_DIRECTIVES = ['label','module','pod','publishDir'] + + protected void putWithRepeat( String name, Object value ) { + if( name in REPEATABLE_DIRECTIVES ) { + config.remove(name) + this.metaClass.invokeMethod(this, name, value) + } + else { + config.put(name, value) + } + } + + /// BUILD + + ProcessConfig getConfig() { + return config + } + + ProcessDef build() { + if ( !body ) + throw new ScriptRuntimeException("Missing script in the specified process block -- make sure it terminates with the script string to be executed") + return new ProcessDef(ownerScript, processName, body, config) + } +} diff --git a/modules/nextflow/src/main/groovy/nextflow/script/dsl/WorkflowDsl.groovy b/modules/nextflow/src/main/groovy/nextflow/script/dsl/WorkflowDsl.groovy new file mode 100644 index 0000000000..1706201c6e --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/script/dsl/WorkflowDsl.groovy @@ -0,0 +1,73 @@ +/* + * Copyright 2013-2023, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.script.dsl + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import nextflow.script.BaseScript +import nextflow.script.BodyDef +import nextflow.script.WorkflowDef +/** + * Implements the workflow DSL. + * + * @author Ben Sherman + */ +@Slf4j +@CompileStatic +class WorkflowDsl { + + static final private String TAKE_PREFIX = '_take_' + static final private String EMIT_PREFIX = '_emit_' + + private BaseScript owner + private String name + private BodyDef body + private Map takes = new LinkedHashMap<>(10) + private Map emits = new LinkedHashMap<>(10) + + WorkflowDsl(BaseScript owner, String name=null) { + this.owner = owner + this.name = name + } + + @Override + def invokeMethod(String name, Object args) { + if( name.startsWith(TAKE_PREFIX) ) + takes.put(name.substring(TAKE_PREFIX.size()), args) + + else if( name.startsWith(EMIT_PREFIX) ) + emits.put(name.substring(EMIT_PREFIX.size()), args) + + else + throw new MissingMethodException(name, WorkflowDef, args) + } + + WorkflowDsl withBody(BodyDef body) { + this.body = body + return this + } + + WorkflowDef build() { + new WorkflowDef( + owner, + name, + body, + new ArrayList<>(takes.keySet()), + new ArrayList<>(emits.keySet()) + ) + } +} diff --git a/modules/nextflow/src/main/groovy/nextflow/util/LoggerHelper.groovy b/modules/nextflow/src/main/groovy/nextflow/util/LoggerHelper.groovy index f78e7c8bd6..d8a9dbe988 100644 --- a/modules/nextflow/src/main/groovy/nextflow/util/LoggerHelper.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/util/LoggerHelper.groovy @@ -65,6 +65,7 @@ import nextflow.script.ChainableDef import nextflow.script.ComponentDef import nextflow.script.CompositeDef import nextflow.script.FunctionDef +import nextflow.script.ProcessDef import nextflow.script.ScriptMeta import nextflow.script.WorkflowBinding import nextflow.script.WorkflowDef @@ -754,7 +755,7 @@ class LoggerHelper { if( type instanceof Class ) { if( DataflowWriteChannel.isAssignableFrom(type) || DataflowReadChannel .isAssignableFrom(type) ) return 'channel type' - if( ComponentDef.isAssignableFrom(type) ) + if( ProcessDef.isAssignableFrom(type) ) return 'process type' if( FunctionDef.isAssignableFrom(type) ) return 'function type' diff --git a/modules/nextflow/src/test/groovy/nextflow/ast/NextflowDSLImplTest.groovy b/modules/nextflow/src/test/groovy/nextflow/ast/DslCodeVisitorTest.groovy similarity index 98% rename from modules/nextflow/src/test/groovy/nextflow/ast/NextflowDSLImplTest.groovy rename to modules/nextflow/src/test/groovy/nextflow/ast/DslCodeVisitorTest.groovy index a8c89e4266..c60308d353 100644 --- a/modules/nextflow/src/test/groovy/nextflow/ast/NextflowDSLImplTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/ast/DslCodeVisitorTest.groovy @@ -13,12 +13,12 @@ import test.MockExecutorFactory * * @author Paolo Di Tommaso */ -class NextflowDSLImplTest extends Dsl2Spec { +class DslCodeVisitorTest extends Dsl2Spec { def createCompilerConfig() { def config = new CompilerConfiguration() config.setScriptBaseClass(BaseScript.class.name) - config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowDSL)) + config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowXform)) return config } diff --git a/modules/nextflow/src/test/groovy/nextflow/ast/OpXformTest.groovy b/modules/nextflow/src/test/groovy/nextflow/ast/OperatorXformTest.groovy similarity index 98% rename from modules/nextflow/src/test/groovy/nextflow/ast/OpXformTest.groovy rename to modules/nextflow/src/test/groovy/nextflow/ast/OperatorXformTest.groovy index f4ea31f681..c04eed7dc8 100644 --- a/modules/nextflow/src/test/groovy/nextflow/ast/OpXformTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/ast/OperatorXformTest.groovy @@ -29,12 +29,12 @@ import org.codehaus.groovy.control.customizers.ASTTransformationCustomizer * * @author Paolo Di Tommaso */ -class OpXformTest extends Specification { +class OperatorXformTest extends Specification { private TokenBranchDef eval_branch(String stmt) { def config = new CompilerConfiguration() - config.addCompilationCustomizers( new ASTTransformationCustomizer(OpXform)) + config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowXform)) def shell = new GroovyShell(config) def result = shell.evaluate(""" @@ -52,7 +52,7 @@ class OpXformTest extends Specification { private TokenMultiMapDef eval_multiMap(String stmt) { def config = new CompilerConfiguration() - config.addCompilationCustomizers( new ASTTransformationCustomizer(OpXform)) + config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowXform)) def shell = new GroovyShell(config) def result = shell.evaluate(""" diff --git a/modules/nextflow/src/test/groovy/nextflow/processor/TaskConfigTest.groovy b/modules/nextflow/src/test/groovy/nextflow/processor/TaskConfigTest.groovy index b08595666f..d76e6055a0 100644 --- a/modules/nextflow/src/test/groovy/nextflow/processor/TaskConfigTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/processor/TaskConfigTest.groovy @@ -102,45 +102,36 @@ class TaskConfigTest extends Specification { given: def config - def local when: - config = new ProcessConfig([:]) - config.module 't_coffee/10' - config.module( [ 'blast/2.2.1', 'clustalw/2'] ) - local = config.createTaskConfig() + config = new TaskConfig() + config.module = ['t_coffee/10', 'blast/2.2.1', 'clustalw/2'] then: - local.module == ['t_coffee/10', 'blast/2.2.1', 'clustalw/2'] - local.getModule() == ['t_coffee/10','blast/2.2.1', 'clustalw/2'] + config.module == ['t_coffee/10', 'blast/2.2.1', 'clustalw/2'] + config.getModule() == ['t_coffee/10','blast/2.2.1', 'clustalw/2'] when: - config = new ProcessConfig([:]) - config.module 'a/1' - config.module 'b/2:c/3' - local = config.createTaskConfig() + config = new TaskConfig() + config.module = ['a/1', 'b/2:c/3'] then: - local.module == ['a/1','b/2','c/3'] + config.module == ['a/1','b/2','c/3'] when: - config = new ProcessConfig([:]) - config.module { 'a/1' } - config.module { 'b/2:c/3' } - config.module 'd/4' - local = config.createTaskConfig() - local.setContext([:]) + config = new TaskConfig() + config.module = ['a/1', 'b/2:c/3', 'd/4'] + config.setContext([:]) then: - local.module == ['a/1','b/2','c/3', 'd/4'] + config.module == ['a/1','b/2','c/3', 'd/4'] when: - config = new ProcessConfig([:]) - config.module = 'b/2:c/3' - local = config.createTaskConfig() + config = new TaskConfig() + config.module = ['b/2:c/3'] then: - local.module == ['b/2','c/3'] - local.getModule() == ['b/2','c/3'] + config.module == ['b/2','c/3'] + config.getModule() == ['b/2','c/3'] } @@ -462,14 +453,13 @@ class TaskConfigTest extends Specification { def 'should create publishDir object' () { setup: - def script = Mock(BaseScript) - ProcessConfig process - PublishDir publish + def config + def publish when: - process = new ProcessConfig(script) - process.publishDir '/data' - publish = process.createTaskConfig().getPublishDir()[0] + config = new TaskConfig() + config.publishDir = [[path: '/data']] + publish = config.getPublishDir()[0] then: publish.path == Paths.get('/data').complete() publish.pattern == null @@ -477,9 +467,9 @@ class TaskConfigTest extends Specification { publish.mode == null when: - process = new ProcessConfig(script) - process.publishDir '/data', overwrite: false, mode: 'copy', pattern: '*.txt' - publish = process.createTaskConfig().getPublishDir()[0] + config = new TaskConfig() + config.publishDir = [[path: '/data', overwrite: false, mode: 'copy', pattern: '*.txt']] + publish = config.getPublishDir()[0] then: publish.path == Paths.get('/data').complete() publish.pattern == '*.txt' @@ -487,18 +477,17 @@ class TaskConfigTest extends Specification { publish.mode == PublishDir.Mode.COPY when: - process = new ProcessConfig(script) - process.publishDir '/my/data', mode: 'copyNoFollow' - publish = process.createTaskConfig().getPublishDir()[0] + config = new TaskConfig() + config.publishDir = [[path: '/my/data', mode: 'copyNoFollow']] + publish = config.getPublishDir()[0] then: publish.path == Paths.get('//my/data').complete() publish.mode == PublishDir.Mode.COPY_NO_FOLLOW when: - process = new ProcessConfig(script) - process.publishDir '/here' - process.publishDir '/there', pattern: '*.fq' - def dirs = process.createTaskConfig().getPublishDir() + config = new TaskConfig() + config.publishDir = [[path: '/here'], [path: '/there', pattern: '*.fq']] + def dirs = config.getPublishDir() then: dirs.size() == 2 dirs[0].path == Paths.get('/here') @@ -549,20 +538,18 @@ class TaskConfigTest extends Specification { def 'should configure pod options'() { - given: - def script = Mock(BaseScript) - when: - def process = new ProcessConfig(script) - process.pod secret: 'foo', mountPath: '/this' - process.pod secret: 'bar', env: 'BAR_XXX' + def config = new TaskConfig() + config.pod = [ + [secret: 'foo', mountPath: '/this'], + [secret: 'bar', env: 'BAR_XXX'] ] then: - process.get('pod') == [ + config.get('pod') == [ [secret: 'foo', mountPath: '/this'], [secret: 'bar', env: 'BAR_XXX'] ] - process.createTaskConfig().getPodOptions() == new PodOptions([ + config.getPodOptions() == new PodOptions([ [secret: 'foo', mountPath: '/this'], [secret: 'bar', env: 'BAR_XXX'] ]) @@ -571,20 +558,19 @@ class TaskConfigTest extends Specification { def 'should get gpu resources' () { given: - def script = Mock(BaseScript) + def config = new TaskConfig() + def res when: - def process = new ProcessConfig(script) - process.accelerator 5 - def res = process.createTaskConfig().getAccelerator() + config.accelerator = [request: 5, limit: 5] + res = config.getAccelerator() then: res.limit == 5 res.request == 5 when: - process = new ProcessConfig(script) - process.accelerator 5, limit: 10, type: 'nvidia' - res = process.createTaskConfig().getAccelerator() + config.accelerator = [request: 5, limit: 10, type: 'nvidia'] + res = config.getAccelerator() then: res.request == 5 res.limit == 10 @@ -593,36 +579,23 @@ class TaskConfigTest extends Specification { def 'should configure secrets'() { - given: - def script = Mock(BaseScript) - when: - def process = new ProcessConfig(script) - process.secret 'alpha' - process.secret 'omega' + def config = new TaskConfig() + config.secret = ['alpha', 'omega'] then: - process.getSecret() == ['alpha', 'omega'] + config.getSecret() == ['alpha', 'omega'] and: - process.createTaskConfig().secret == ['alpha', 'omega'] - process.createTaskConfig().getSecret() == ['alpha', 'omega'] + config.secret == ['alpha', 'omega'] + config.getSecret() == ['alpha', 'omega'] } def 'should configure resourceLabels options'() { - given: - def script = Mock(BaseScript) - - when: - def process = new ProcessConfig(script) - process.resourceLabels( region: 'eu-west-1', organization: 'A', user: 'this', team: 'that' ) - - then: - process.get('resourceLabels') == [region: 'eu-west-1', organization: 'A', user: 'this', team: 'that'] - when: - def config = process.createTaskConfig() + def config = new TaskConfig() + config.resourceLabels = [region: 'eu-west-1', organization: 'A', user: 'this', team: 'that'] then: config.getResourceLabels() == [region: 'eu-west-1', organization: 'A', user: 'this', team: 'that'] config.getResourceLabelsAsString() == 'region=eu-west-1,organization=A,user=this,team=that' diff --git a/modules/nextflow/src/test/groovy/nextflow/processor/TaskProcessorTest.groovy b/modules/nextflow/src/test/groovy/nextflow/processor/TaskProcessorTest.groovy index b616669c2e..783cfa0848 100644 --- a/modules/nextflow/src/test/groovy/nextflow/processor/TaskProcessorTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/processor/TaskProcessorTest.groovy @@ -109,7 +109,7 @@ class TaskProcessorTest extends Specification { when: def session = new Session([env: [X:"1", Y:"2"]]) session.setBaseDir(home) - def processor = new DummyProcessor('task1', session, Mock(BaseScript), Mock(ProcessConfig)) + def processor = new DummyProcessor('task1', session, Mock(BaseScript), new ProcessConfig([:])) def builder = new ProcessBuilder() builder.environment().putAll( processor.getProcessEnvironment() ) then: @@ -121,7 +121,7 @@ class TaskProcessorTest extends Specification { when: session = new Session([env: [X:"1", Y:"2", PATH:'/some']]) session.setBaseDir(home) - processor = new DummyProcessor('task1', session, Mock(BaseScript), Mock(ProcessConfig)) + processor = new DummyProcessor('task1', session, Mock(BaseScript), new ProcessConfig([:])) builder = new ProcessBuilder() builder.environment().putAll( processor.getProcessEnvironment() ) then: diff --git a/modules/nextflow/src/test/groovy/nextflow/scm/ProviderConfigTest.groovy b/modules/nextflow/src/test/groovy/nextflow/scm/ProviderConfigTest.groovy index c8779571f8..23fe7dc9d8 100644 --- a/modules/nextflow/src/test/groovy/nextflow/scm/ProviderConfigTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/scm/ProviderConfigTest.groovy @@ -16,6 +16,8 @@ package nextflow.scm +import java.nio.file.Files + import spock.lang.Specification import spock.lang.Unroll @@ -242,4 +244,30 @@ class ProviderConfigTest extends Specification { 'paolo0758/nf-azure-repo/_git/nf-azure-repo' | 'https://dev.azure.com' | 'paolo0758/nf-azure-repo' } + def 'should get default config path' () { + given: + ProviderConfig.env.remove('NXF_SCM_FILE') + + when: + def path = ProviderConfig.getScmConfigPath() + then: + path.toString() == "${System.getProperty('user.home')}/.nextflow/scm" + + } + + def 'should get custom config path' () { + given: + def cfg = Files.createTempFile('test','config') + ProviderConfig.env.NXF_SCM_FILE = cfg.toString() + + when: + def path = ProviderConfig.getScmConfigPath() + then: + path.toString() == cfg.toString() + + cleanup: + ProviderConfig.env.remove('NXF_SCM_FILE') + cfg.delete() + } + } diff --git a/modules/nextflow/src/test/groovy/nextflow/script/IncludeDefTest.groovy b/modules/nextflow/src/test/groovy/nextflow/script/IncludeDefTest.groovy index 01b1b83164..ab81df37e4 100644 --- a/modules/nextflow/src/test/groovy/nextflow/script/IncludeDefTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/script/IncludeDefTest.groovy @@ -7,7 +7,7 @@ import spock.lang.Unroll import java.nio.file.NoSuchFileException import java.nio.file.Path -import nextflow.ast.NextflowDSL +import nextflow.ast.NextflowXform import nextflow.exception.IllegalModulePath import nextflow.file.FileHelper import org.codehaus.groovy.control.CompilerConfiguration @@ -174,7 +174,7 @@ class IncludeDefTest extends Specification { def binding = new ScriptBinding([params: [foo:1, bar:2]]) def config = new CompilerConfiguration() config.setScriptBaseClass(TestScript.class.name) - config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowDSL)) + config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowXform)) when: def script = (TestScript)new GroovyShell(binding, config).parse(INCLUDE) diff --git a/modules/nextflow/src/test/groovy/nextflow/script/ProcessConfigTest.groovy b/modules/nextflow/src/test/groovy/nextflow/script/ProcessConfigTest.groovy index a9d8a108d8..26f11ce269 100644 --- a/modules/nextflow/src/test/groovy/nextflow/script/ProcessConfigTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/script/ProcessConfigTest.groovy @@ -16,20 +16,11 @@ package nextflow.script -import java.nio.file.Files - -import nextflow.scm.ProviderConfig import spock.lang.Specification import spock.lang.Unroll -import nextflow.exception.IllegalDirectiveException import nextflow.processor.ErrorStrategy -import nextflow.script.params.FileInParam -import nextflow.script.params.StdInParam -import nextflow.script.params.StdOutParam -import nextflow.script.params.ValueInParam import nextflow.util.Duration -import nextflow.util.MemoryUnit import static nextflow.util.CacheHelper.HashMode /** * @@ -64,45 +55,12 @@ class ProcessConfigTest extends Specification { then: config.tag == 'val 1' - // setting list values - when: - config.tag 1,2,3 - then: - config.tag == [1,2,3] - - // setting named parameters attribute - when: - config.tag field1:'val1', field2: 'val2' - then: - config.tag == [field1:'val1', field2: 'val2'] - // generic value assigned like a 'plain' property when: config.tag = 99 then: config.tag == 99 - // maxDuration property - when: - config.time '1h' - then: - config.time == '1h' - config.createTaskConfig().time == new Duration('1h') - - // maxMemory property - when: - config.memory '2GB' - then: - config.memory == '2GB' - config.createTaskConfig().memory == new MemoryUnit('2GB') - - when: - config.stageInMode 'copy' - config.stageOutMode 'move' - then: - config.stageInMode == 'copy' - config.stageOutMode == 'move' - } @Unroll @@ -145,16 +103,6 @@ class ProcessConfigTest extends Specification { } - def 'should throw MissingPropertyException' () { - when: - def script = Mock(BaseScript) - def config = new ProcessConfig(script).throwExceptionOnMissingProperty(true) - def x = config.hola - - then: - thrown(MissingPropertyException) - } - def 'should check property existence' () { @@ -171,59 +119,6 @@ class ProcessConfigTest extends Specification { } - def 'should create input directives' () { - - setup: - def script = Mock(BaseScript) - def config = new ProcessConfig(script) - - when: - config._in_file([infile:'filename.fa']) - config._in_val('x').setFrom(1) - config._in_stdin() - - then: - config.getInputs().size() == 3 - - config.inputs.get(0) instanceof FileInParam - config.inputs.get(0).name == 'infile' - (config.inputs.get(0) as FileInParam).filePattern == 'filename.fa' - - config.inputs.get(1) instanceof ValueInParam - config.inputs.get(1).name == 'x' - - config.inputs.get(2).name == '-' - config.inputs.get(2) instanceof StdInParam - - config.inputs.names == [ 'infile', 'x', '-' ] - config.inputs.ofType( FileInParam ) == [ config.getInputs().get(0) ] - - } - - def 'should create output directives' () { - - setup: - def script = Mock(BaseScript) - def config = new ProcessConfig(script) - - when: - config._out_stdout() - config._out_file(new TokenVar('file1')).setInto('ch1') - config._out_file(new TokenVar('file2')).setInto('ch2') - config._out_file(new TokenVar('file3')).setInto('ch3') - - then: - config.outputs.size() == 4 - config.outputs.names == ['-', 'file1', 'file2', 'file3'] - config.outputs.ofType(StdOutParam).size() == 1 - - config.outputs[0] instanceof StdOutParam - config.outputs[1].name == 'file1' - config.outputs[2].name == 'file2' - config.outputs[3].name == 'file3' - - } - def 'should set cache attribute'() { @@ -273,542 +168,4 @@ class ProcessConfigTest extends Specification { } - def 'should create PublishDir object' () { - - setup: - BaseScript script = Mock(BaseScript) - ProcessConfig config - - when: - config = new ProcessConfig(script) - config.publishDir '/data' - then: - config.get('publishDir')[0] == [path:'/data'] - - when: - config = new ProcessConfig(script) - config.publishDir '/data', mode: 'link', pattern: '*.bam' - then: - config.get('publishDir')[0] == [path: '/data', mode: 'link', pattern: '*.bam'] - - when: - config = new ProcessConfig(script) - config.publishDir path: '/data', mode: 'link', pattern: '*.bam' - then: - config.get('publishDir')[0] == [path: '/data', mode: 'link', pattern: '*.bam'] - } - - def 'should throw InvalidDirectiveException'() { - - given: - def script = Mock(BaseScript) - def config = new ProcessConfig(script) - - when: - config.hello 'world' - - then: - def e = thrown(IllegalDirectiveException) - e.message == - ''' - Unknown process directive: `hello` - - Did you mean of these? - shell - ''' - .stripIndent().trim() - } - - def 'should set process secret'() { - when: - def config = new ProcessConfig([:]) - then: - config.getSecret() == [] - - when: - config.secret('foo') - then: - config.getSecret() == ['foo'] - - when: - config.secret('bar') - then: - config.secret == ['foo', 'bar'] - config.getSecret() == ['foo', 'bar'] - } - - def 'should set process labels'() { - when: - def config = new ProcessConfig([:]) - then: - config.getLabels() == [] - - when: - config.label('foo') - then: - config.getLabels() == ['foo'] - - when: - config.label('bar') - then: - config.getLabels() == ['foo','bar'] - } - - def 'should apply resource labels config' () { - given: - def config = new ProcessConfig(Mock(BaseScript)) - expect: - config.getResourceLabels() == [:] - - when: - config.resourceLabels([foo: 'one', bar: 'two']) - then: - config.getResourceLabels() == [foo: 'one', bar: 'two'] - - when: - config.resourceLabels([foo: 'new one', baz: 'three']) - then: - config.getResourceLabels() == [foo: 'new one', bar: 'two', baz: 'three'] - - } - - def 'should check a valid label' () { - - expect: - new ProcessConfig([:]).isValidLabel(lbl) == result - - where: - lbl | result - 'foo' | true - 'foo1' | true - '1foo' | false - '_foo' | false - 'foo1_' | false - 'foo_1' | true - 'foo-1' | false - 'foo.1' | false - 'a' | true - 'A' | true - '1' | false - '_' | false - 'a=b' | true - 'a=foo' | true - 'a=foo_1' | true - 'a=foo_' | false - '_=foo' | false - '=a' | false - 'a=' | false - 'a=1' | false - - } - - @Unroll - def 'should match selector: #SELECTOR with #TARGET' () { - expect: - ProcessConfig.matchesSelector(TARGET, SELECTOR) == EXPECTED - - where: - SELECTOR | TARGET | EXPECTED - 'foo' | 'foo' | true - 'foo' | 'bar' | false - '!foo' | 'bar' | true - 'a|b' | 'a' | true - 'a|b' | 'b' | true - 'a|b' | 'z' | false - 'a*' | 'a' | true - 'a*' | 'aaaa' | true - 'a*' | 'bbbb' | false - } - - def 'should apply config setting for a process label' () { - given: - def settings = [ - 'withLabel:short' : [ cpus: 1, time: '1h'], - 'withLabel:!short' : [ cpus: 32, queue: 'cn-long'], - 'withLabel:foo' : [ cpus: 2 ], - 'withLabel:foo|bar': [ disk: '100GB' ], - 'withLabel:gpu.+' : [ cpus: 4 ], - ] - - when: - def process = new ProcessConfig([:]) - process.applyConfigSelectorWithLabels(settings, ['short']) - then: - process.cpus == 1 - process.time == '1h' - process.size() == 2 - - when: - process = new ProcessConfig([:]) - process.applyConfigSelectorWithLabels(settings, ['long']) - then: - process.cpus == 32 - process.queue == 'cn-long' - process.size() == 2 - - when: - process = new ProcessConfig([:]) - process.applyConfigSelectorWithLabels(settings, ['foo']) - then: - process.cpus == 2 - process.disk == '100GB' - process.queue == 'cn-long' - process.size() == 3 - - when: - process = new ProcessConfig([:]) - process.applyConfigSelectorWithLabels(settings, ['bar']) - then: - process.cpus == 32 - process.disk == '100GB' - process.queue == 'cn-long' - process.size() == 3 - - when: - process = new ProcessConfig([:]) - process.applyConfigSelectorWithLabels(settings, ['gpu-1']) - then: - process.cpus == 4 - process.queue == 'cn-long' - process.size() == 2 - - } - - - def 'should apply config setting for a process name' () { - given: - def settings = [ - 'withName:alpha' : [ cpus: 1, time: '1h'], - 'withName:delta' : [ cpus: 2 ], - 'withName:delta|gamma' : [ disk: '100GB' ], - 'withName:omega.+' : [ cpus: 4 ], - ] - - when: - def process = new ProcessConfig([:]) - process.applyConfigSelectorWithName(settings, 'xx') - then: - process.size() == 0 - - when: - process = new ProcessConfig([:]) - process.applyConfigSelectorWithName(settings, 'alpha') - then: - process.cpus == 1 - process.time == '1h' - process.size() == 2 - - when: - process = new ProcessConfig([:]) - process.applyConfigSelectorWithName(settings, 'delta') - then: - process.cpus == 2 - process.disk == '100GB' - process.size() == 2 - - when: - process = new ProcessConfig([:]) - process.applyConfigSelectorWithName(settings, 'gamma') - then: - process.disk == '100GB' - process.size() == 1 - - when: - process = new ProcessConfig([:]) - process.applyConfigSelectorWithName(settings, 'omega_x') - then: - process.cpus == 4 - process.size() == 1 - } - - - def 'should apply config process defaults' () { - - when: - def process = new ProcessConfig(Mock(BaseScript)) - - // set process specific settings - process.queue = 'cn-el6' - process.memory = '10 GB' - - // apply config defaults - process.applyConfigDefaults( - queue: 'def-queue', - container: 'ubuntu:latest' - ) - - then: - process.queue == 'cn-el6' - process.container == 'ubuntu:latest' - process.memory == '10 GB' - process.cacheable == true - - - - when: - process = new ProcessConfig(Mock(BaseScript)) - // set process specific settings - process.container = null - // apply process defaults - process.applyConfigDefaults( - queue: 'def-queue', - container: 'ubuntu:latest', - maxRetries: 5 - ) - then: - process.queue == 'def-queue' - process.container == null - process.maxRetries == 5 - - - - when: - process = new ProcessConfig(Mock(BaseScript)) - // set process specific settings - process.maxRetries = 10 - // apply process defaults - process.applyConfigDefaults( - queue: 'def-queue', - container: 'ubuntu:latest', - maxRetries: 5 - ) - then: - process.queue == 'def-queue' - process.container == 'ubuntu:latest' - process.maxRetries == 10 - } - - - def 'should apply pod configs' () { - - when: - def process = new ProcessConfig([:]) - process.applyConfigDefaults( pod: [secret: 'foo', mountPath: '/there'] ) - then: - process.pod == [ - [secret: 'foo', mountPath: '/there'] - ] - - when: - process = new ProcessConfig([:]) - process.applyConfigDefaults( pod: [ - [secret: 'foo', mountPath: '/here'], - [secret: 'bar', mountPath: '/there'] - ] ) - - then: - process.pod == [ - [secret: 'foo', mountPath: '/here'], - [secret: 'bar', mountPath: '/there'] - ] - - } - - def 'should clone config object' () { - - given: - def config = new ProcessConfig(Mock(BaseScript)) - - when: - config.queue 'cn-el6' - config.container 'ubuntu:latest' - config.memory '10 GB' - config._in_val('foo') - config._in_file('sample.txt') - config._out_file('result.txt') - - then: - config.queue == 'cn-el6' - config.container == 'ubuntu:latest' - config.memory == '10 GB' - config.getInputs().size() == 2 - config.getOutputs().size() == 1 - - when: - def copy = config.clone() - copy.queue 'long' - copy.container 'debian:wheezy' - copy.memory '5 GB' - copy._in_val('bar') - copy._out_file('sample.bam') - - then: - copy.queue == 'long' - copy.container == 'debian:wheezy' - copy.memory == '5 GB' - copy.getInputs().size() == 3 - copy.getOutputs().size() == 2 - - // original config is not affected - config.queue == 'cn-el6' - config.container == 'ubuntu:latest' - config.memory == '10 GB' - config.getInputs().size() == 2 - config.getOutputs().size() == 1 - } - - def 'should apply accelerator config' () { - - given: - def process = new ProcessConfig(Mock(BaseScript)) - - when: - process.accelerator 5 - then: - process.accelerator == [limit: 5] - - when: - process.accelerator request: 1, limit: 5, type: 'nvida' - then: - process.accelerator == [request: 1, limit: 5, type: 'nvida'] - - when: - process.accelerator 5, type: 'nvida' - then: - process.accelerator == [limit: 5, type: 'nvida'] - - when: - process.accelerator 1, limit: 5 - then: - process.accelerator == [request: 1, limit:5] - - when: - process.accelerator 5, request: 1 - then: - process.accelerator == [request: 1, limit:5] - } - - def 'should apply disk config' () { - - given: - def process = new ProcessConfig(Mock(BaseScript)) - - when: - process.disk '100 GB' - then: - process.disk == [request: '100 GB'] - - when: - process.disk '375 GB', type: 'local-ssd' - then: - process.disk == [request: '375 GB', type: 'local-ssd'] - - when: - process.disk request: '375 GB', type: 'local-ssd' - then: - process.disk == [request: '375 GB', type: 'local-ssd'] - } - - def 'should apply architecture config' () { - - given: - def process = new ProcessConfig(Mock(BaseScript)) - - when: - process.arch 'linux/x86_64' - then: - process.arch == [name: 'linux/x86_64'] - - when: - process.arch 'linux/x86_64', target: 'zen3' - then: - process.arch == [name: 'linux/x86_64', target: 'zen3'] - - when: - process.arch name: 'linux/x86_64', target: 'zen3' - then: - process.arch == [name: 'linux/x86_64', target: 'zen3'] - } - - - def 'should get default config path' () { - given: - ProviderConfig.env.remove('NXF_SCM_FILE') - - when: - def path = ProviderConfig.getScmConfigPath() - then: - path.toString() == "${System.getProperty('user.home')}/.nextflow/scm" - - } - - def 'should get custom config path' () { - given: - def cfg = Files.createTempFile('test','config') - ProviderConfig.env.NXF_SCM_FILE = cfg.toString() - - when: - def path = ProviderConfig.getScmConfigPath() - then: - path.toString() == cfg.toString() - - cleanup: - ProviderConfig.env.remove('NXF_SCM_FILE') - cfg.delete() - } - - def 'should not apply config on negative label' () { - given: - def settings = [ - 'withLabel:foo': [ cpus: 2 ], - 'withLabel:!foo': [ cpus: 4 ], - 'withLabel:!nodisk_.*': [ disk: '100.GB'] - ] - - when: - def p1 = new ProcessConfig([label: ['foo', 'other']]) - p1.applyConfig(settings, "processName", null, null) - then: - p1.cpus == 2 - p1.disk == '100.GB' - - when: - def p2 = new ProcessConfig([label: ['foo', 'other', 'nodisk_label']]) - p2.applyConfig(settings, "processName", null, null) - then: - p2.cpus == 2 - !p2.disk - - when: - def p3 = new ProcessConfig([label: ['other', 'nodisk_label']]) - p3.applyConfig(settings, "processName", null, null) - then: - p3.cpus == 4 - !p3.disk - - } - - def 'should throw exception for invalid error strategy' () { - when: - def process1 = new ProcessConfig(Mock(BaseScript)) - process1.errorStrategy 'abort' - - then: - def e1 = thrown(IllegalArgumentException) - e1.message == "Unknown error strategy 'abort' ― Available strategies are: terminate,finish,ignore,retry" - - } - - def 'should not throw exception for valid error strategy or closure' () { - when: - def process1 = new ProcessConfig(Mock(BaseScript)) - process1.errorStrategy 'retry' - - then: - def e1 = noExceptionThrown() - - when: - def process2 = new ProcessConfig(Mock(BaseScript)) - process2.errorStrategy 'terminate' - - then: - def e2 = noExceptionThrown() - - when: - def process3 = new ProcessConfig(Mock(BaseScript)) - process3.errorStrategy { task.exitStatus==14 ? 'retry' : 'terminate' } - - then: - def e3 = noExceptionThrown() - } } diff --git a/modules/nextflow/src/test/groovy/nextflow/script/ProcessDefTest.groovy b/modules/nextflow/src/test/groovy/nextflow/script/ProcessDefTest.groovy index 6b8fc4805a..131d60786d 100644 --- a/modules/nextflow/src/test/groovy/nextflow/script/ProcessDefTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/script/ProcessDefTest.groovy @@ -12,8 +12,8 @@ class ProcessDefTest extends Specification { given: def OWNER = Mock(BaseScript) - def BODY = { -> null } - def proc = new ProcessDef(OWNER, BODY, 'foo') + def BODY = new BodyDef({->}, 'echo hello') + def proc = new ProcessDef(OWNER, 'foo', BODY, new ProcessConfig([:])) when: def copy = proc.cloneWithName('foo_alias') @@ -22,8 +22,8 @@ class ProcessDefTest extends Specification { copy.getSimpleName() == 'foo_alias' copy.getBaseName() == 'foo' copy.getOwner() == OWNER - copy.rawBody.class == BODY.class - !copy.rawBody.is(BODY) + copy.taskBody.class == BODY.class + !copy.taskBody.is(BODY) when: copy = proc.cloneWithName('flow1:flow2:foo') @@ -32,8 +32,8 @@ class ProcessDefTest extends Specification { copy.getSimpleName() == 'foo' copy.getBaseName() == 'foo' copy.getOwner() == OWNER - copy.rawBody.class == BODY.class - !copy.rawBody.is(BODY) + copy.taskBody.class == BODY.class + !copy.taskBody.is(BODY) } def 'should apply process config' () { @@ -47,11 +47,11 @@ class ProcessDefTest extends Specification { 'withName:flow1:flow2:flow3:bar': [memory: '8GB'] ] ] - def BODY = {-> - return new BodyDef({->}, 'echo hello') + def BODY = new BodyDef({->}, 'echo hello') + def proc = new ProcessDef(OWNER, 'foo', BODY, new ProcessConfig([:])) + proc.session = Mock(Session) { + getConfig() >> CONFIG } - def proc = new ProcessDef(OWNER, BODY, 'foo') - proc.session = Mock(Session) { getConfig() >> CONFIG } when: def copy = proc.clone() diff --git a/modules/nextflow/src/test/groovy/nextflow/script/ScriptMetaTest.groovy b/modules/nextflow/src/test/groovy/nextflow/script/ScriptMetaTest.groovy index d94be89795..72de20f3cb 100644 --- a/modules/nextflow/src/test/groovy/nextflow/script/ScriptMetaTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/script/ScriptMetaTest.groovy @@ -45,8 +45,8 @@ class ScriptMetaTest extends Dsl2Spec { given: def script = new FooScript(new ScriptBinding()) - def proc1 = new ProcessDef(script, Mock(Closure), 'proc1') - def proc2 = new ProcessDef(script, Mock(Closure), 'proc2') + def proc1 = new ProcessDef(script, 'proc1', Mock(BodyDef), new ProcessConfig([:])) + def proc2 = new ProcessDef(script, 'proc2', Mock(BodyDef), new ProcessConfig([:])) def func1 = new FunctionDef(name: 'func1', alias: 'func1') def work1 = new WorkflowDef(name:'work1') @@ -83,19 +83,19 @@ class ScriptMetaTest extends Dsl2Spec { // defs in the root script def func1 = new FunctionDef(name: 'func1', alias: 'func1') - def proc1 = new ProcessDef(script1, Mock(Closure), 'proc1') + def proc1 = new ProcessDef(script1, 'proc1', Mock(BodyDef), new ProcessConfig([:])) def work1 = new WorkflowDef(name:'work1') meta1.addDefinition(proc1, func1, work1) // defs in the second script imported in the root namespace def func2 = new FunctionDef(name: 'func2', alias: 'func2') - def proc2 = new ProcessDef(script2, Mock(Closure), 'proc2') + def proc2 = new ProcessDef(script2, 'proc2', Mock(BodyDef), new ProcessConfig([:])) def work2 = new WorkflowDef(name:'work2') meta2.addDefinition(proc2, func2, work2) // defs in the third script imported in a separate namespace def func3 = new FunctionDef(name: 'func3', alias: 'func3') - def proc3 = new ProcessDef(script2, Mock(Closure), 'proc3') + def proc3 = new ProcessDef(script2, 'proc3', Mock(BodyDef), new ProcessConfig([:])) def work3 = new WorkflowDef(name:'work3') meta3.addDefinition(proc3, func3, work3) @@ -205,7 +205,7 @@ class ScriptMetaTest extends Dsl2Spec { // import module into main script def func2 = new FunctionDef(name: 'func1', alias: 'func1') - def proc2 = new ProcessDef(script2, Mock(Closure), 'proc1') + def proc2 = new ProcessDef(script2, 'proc1', Mock(BodyDef), new ProcessConfig([:])) def work2 = new WorkflowDef(name: 'work1') meta2.addDefinition(proc2, func2, work2) @@ -215,7 +215,7 @@ class ScriptMetaTest extends Dsl2Spec { // attempt to define duplicate components in main script def func1 = new FunctionDef(name: 'func1', alias: 'func1') - def proc1 = new ProcessDef(script1, Mock(Closure), 'proc1') + def proc1 = new ProcessDef(script1, 'proc1', Mock(BodyDef), new ProcessConfig([:])) def work1 = new WorkflowDef(name: 'work1') when: diff --git a/modules/nextflow/src/test/groovy/nextflow/script/WorkflowDefTest.groovy b/modules/nextflow/src/test/groovy/nextflow/script/WorkflowDefTest.groovy index 00a0f9dedf..4aa8687bf9 100644 --- a/modules/nextflow/src/test/groovy/nextflow/script/WorkflowDefTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/script/WorkflowDefTest.groovy @@ -6,7 +6,7 @@ import groovy.util.logging.Slf4j import groovyx.gpars.dataflow.DataflowQueue import groovyx.gpars.dataflow.DataflowVariable import nextflow.Session -import nextflow.ast.NextflowDSL +import nextflow.ast.NextflowXform import org.codehaus.groovy.control.CompilerConfiguration import org.codehaus.groovy.control.MultipleCompilationErrorsException import org.codehaus.groovy.control.customizers.ASTTransformationCustomizer @@ -50,7 +50,7 @@ class WorkflowDefTest extends Dsl2Spec { given: def config = new CompilerConfiguration() config.setScriptBaseClass(TestScript.class.name) - config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowDSL)) + config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowXform)) def SCRIPT = ''' @@ -137,7 +137,7 @@ class WorkflowDefTest extends Dsl2Spec { given: def config = new CompilerConfiguration() config.setScriptBaseClass(TestScript.class.name) - config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowDSL)) + config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowXform)) def SCRIPT = ''' @@ -167,7 +167,7 @@ class WorkflowDefTest extends Dsl2Spec { given: def config = new CompilerConfiguration() config.setScriptBaseClass(TestScript.class.name) - config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowDSL)) + config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowXform)) def SCRIPT = ''' @@ -192,7 +192,7 @@ class WorkflowDefTest extends Dsl2Spec { given: def config = new CompilerConfiguration() config.setScriptBaseClass(TestScript.class.name) - config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowDSL)) + config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowXform)) def SCRIPT = ''' @@ -214,7 +214,7 @@ class WorkflowDefTest extends Dsl2Spec { given: def config = new CompilerConfiguration() config.setScriptBaseClass(TestScript.class.name) - config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowDSL)) + config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowXform)) def SCRIPT = ''' @@ -318,7 +318,7 @@ class WorkflowDefTest extends Dsl2Spec { given: def config = new CompilerConfiguration() config.setScriptBaseClass(TestScript.class.name) - config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowDSL)) + config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowXform)) def SCRIPT = ''' @@ -351,7 +351,7 @@ class WorkflowDefTest extends Dsl2Spec { given: def config = new CompilerConfiguration() config.setScriptBaseClass(TestScript.class.name) - config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowDSL)) + config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowXform)) def SCRIPT = ''' workflow foo { } diff --git a/modules/nextflow/src/test/groovy/nextflow/script/dsl/ProcessDslTest.groovy b/modules/nextflow/src/test/groovy/nextflow/script/dsl/ProcessDslTest.groovy new file mode 100644 index 0000000000..3ba06d411b --- /dev/null +++ b/modules/nextflow/src/test/groovy/nextflow/script/dsl/ProcessDslTest.groovy @@ -0,0 +1,640 @@ +/* + * Copyright 2013-2023, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.script.dsl + +import spock.lang.Specification +import spock.lang.Unroll + +import nextflow.exception.IllegalDirectiveException +import nextflow.script.params.FileInParam +import nextflow.script.params.StdInParam +import nextflow.script.params.StdOutParam +import nextflow.script.params.ValueInParam +import nextflow.script.BaseScript +import nextflow.script.ProcessConfig +import nextflow.script.TokenVar +import nextflow.util.Duration +import nextflow.util.MemoryUnit +/** + * + * @author Paolo Di Tommaso + */ +class ProcessDslTest extends Specification { + + def 'should set directives' () { + + setup: + def builder = new ProcessDsl(Mock(BaseScript), null) + def config = builder.getConfig() + + // setting list values + when: + builder.tag 1,2,3 + then: + config.tag == [1,2,3] + + // setting named parameters attribute + when: + builder.tag field1:'val1', field2: 'val2' + then: + config.tag == [field1:'val1', field2: 'val2'] + + // maxDuration property + when: + builder.time '1h' + then: + config.time == '1h' + config.createTaskConfig().time == new Duration('1h') + + // maxMemory property + when: + builder.memory '2GB' + then: + config.memory == '2GB' + config.createTaskConfig().memory == new MemoryUnit('2GB') + + when: + builder.stageInMode 'copy' + builder.stageOutMode 'move' + then: + config.stageInMode == 'copy' + config.stageOutMode == 'move' + + } + + + def 'should create input directives' () { + + setup: + def builder = new ProcessDsl(Mock(BaseScript), null) + def config = builder.getConfig() + + when: + builder._in_file([infile:'filename.fa']) + builder._in_val('x').setFrom(1) + builder._in_stdin() + + then: + config.getInputs().size() == 3 + + config.inputs.get(0) instanceof FileInParam + config.inputs.get(0).name == 'infile' + (config.inputs.get(0) as FileInParam).filePattern == 'filename.fa' + + config.inputs.get(1) instanceof ValueInParam + config.inputs.get(1).name == 'x' + + config.inputs.get(2).name == '-' + config.inputs.get(2) instanceof StdInParam + + config.inputs.names == [ 'infile', 'x', '-' ] + config.inputs.ofType( FileInParam ) == [ config.getInputs().get(0) ] + + } + + def 'should create output directives' () { + + setup: + def builder = new ProcessDsl(Mock(BaseScript), null) + def config = builder.getConfig() + + when: + builder._out_stdout() + builder._out_file(new TokenVar('file1')).setInto('ch1') + builder._out_file(new TokenVar('file2')).setInto('ch2') + builder._out_file(new TokenVar('file3')).setInto('ch3') + + then: + config.outputs.size() == 4 + config.outputs.names == ['-', 'file1', 'file2', 'file3'] + config.outputs.ofType(StdOutParam).size() == 1 + + config.outputs[0] instanceof StdOutParam + config.outputs[1].name == 'file1' + config.outputs[2].name == 'file2' + config.outputs[3].name == 'file3' + + } + + def 'should create PublishDir object' () { + + setup: + def builder = new ProcessDsl(Mock(BaseScript), null) + def config = builder.getConfig() + + when: + builder.publishDir '/data' + then: + config.get('publishDir').last() == [path:'/data'] + + when: + builder.publishDir '/data', mode: 'link', pattern: '*.bam' + then: + config.get('publishDir').last() == [path: '/data', mode: 'link', pattern: '*.bam'] + + when: + builder.publishDir path: '/data', mode: 'link', pattern: '*.bam' + then: + config.get('publishDir').last() == [path: '/data', mode: 'link', pattern: '*.bam'] + } + + def 'should throw IllegalDirectiveException'() { + + given: + def builder = new ProcessDsl(Mock(BaseScript), null) + + when: + builder.hello 'world' + + then: + def e = thrown(IllegalDirectiveException) + e.message == + ''' + Unknown process directive: `hello` + + Did you mean one of these? + shell + ''' + .stripIndent().trim() + } + + def 'should set process secret'() { + when: + def builder = new ProcessDsl(Mock(BaseScript), null) + def config = builder.getConfig() + then: + config.getSecret() == [] + + when: + builder.secret 'foo' + then: + config.getSecret() == ['foo'] + + when: + builder.secret 'bar' + then: + config.secret == ['foo', 'bar'] + config.getSecret() == ['foo', 'bar'] + } + + def 'should set process labels'() { + when: + def builder = new ProcessDsl(Mock(BaseScript), null) + def config = builder.getConfig() + then: + config.getLabels() == [] + + when: + builder.label 'foo' + then: + config.getLabels() == ['foo'] + + when: + builder.label 'bar' + then: + config.getLabels() == ['foo','bar'] + } + + def 'should apply resource labels config' () { + given: + def builder = new ProcessDsl(Mock(BaseScript), null) + def config = builder.getConfig() + expect: + config.getResourceLabels() == [:] + + when: + builder.resourceLabels foo: 'one', bar: 'two' + then: + config.getResourceLabels() == [foo: 'one', bar: 'two'] + + when: + builder.resourceLabels foo: 'new one', baz: 'three' + then: + config.getResourceLabels() == [foo: 'new one', bar: 'two', baz: 'three'] + + } + + def 'should check a valid label' () { + + expect: + ProcessDsl.isValidLabel(lbl) == result + + where: + lbl | result + 'foo' | true + 'foo1' | true + '1foo' | false + '_foo' | false + 'foo1_' | false + 'foo_1' | true + 'foo-1' | false + 'foo.1' | false + 'a' | true + 'A' | true + '1' | false + '_' | false + 'a=b' | true + 'a=foo' | true + 'a=foo_1' | true + 'a=foo_' | false + '_=foo' | false + '=a' | false + 'a=' | false + 'a=1' | false + + } + + @Unroll + def 'should match selector: #SELECTOR with #TARGET' () { + expect: + ProcessDsl.matchesSelector(TARGET, SELECTOR) == EXPECTED + + where: + SELECTOR | TARGET | EXPECTED + 'foo' | 'foo' | true + 'foo' | 'bar' | false + '!foo' | 'bar' | true + 'a|b' | 'a' | true + 'a|b' | 'b' | true + 'a|b' | 'z' | false + 'a*' | 'a' | true + 'a*' | 'aaaa' | true + 'a*' | 'bbbb' | false + } + + def 'should apply config setting for a process label' () { + given: + def settings = [ + 'withLabel:short' : [ cpus: 1, time: '1h'], + 'withLabel:!short' : [ cpus: 32, queue: 'cn-long'], + 'withLabel:foo' : [ cpus: 2 ], + 'withLabel:foo|bar': [ disk: '100GB' ], + 'withLabel:gpu.+' : [ cpus: 4 ], + ] + + when: + def config = new ProcessConfig([:]) + new ProcessDsl(config).applyConfigSelectorWithLabels(settings, ['short']) + then: + config.cpus == 1 + config.time == '1h' + config.size() == 2 + + when: + config = new ProcessConfig([:]) + new ProcessDsl(config).applyConfigSelectorWithLabels(settings, ['long']) + then: + config.cpus == 32 + config.queue == 'cn-long' + config.size() == 2 + + when: + config = new ProcessConfig([:]) + new ProcessDsl(config).applyConfigSelectorWithLabels(settings, ['foo']) + then: + config.cpus == 2 + config.disk == '100GB' + config.queue == 'cn-long' + config.size() == 3 + + when: + config = new ProcessConfig([:]) + new ProcessDsl(config).applyConfigSelectorWithLabels(settings, ['bar']) + then: + config.cpus == 32 + config.disk == '100GB' + config.queue == 'cn-long' + config.size() == 3 + + when: + config = new ProcessConfig([:]) + new ProcessDsl(config).applyConfigSelectorWithLabels(settings, ['gpu-1']) + then: + config.cpus == 4 + config.queue == 'cn-long' + config.size() == 2 + + } + + + def 'should apply config setting for a process name' () { + given: + def settings = [ + 'withName:alpha' : [ cpus: 1, time: '1h'], + 'withName:delta' : [ cpus: 2 ], + 'withName:delta|gamma' : [ disk: '100GB' ], + 'withName:omega.+' : [ cpus: 4 ], + ] + + when: + def config = new ProcessConfig([:]) + new ProcessDsl(config).applyConfigSelectorWithName(settings, 'xx') + then: + config.size() == 0 + + when: + config = new ProcessConfig([:]) + new ProcessDsl(config).applyConfigSelectorWithName(settings, 'alpha') + then: + config.cpus == 1 + config.time == '1h' + config.size() == 2 + + when: + config = new ProcessConfig([:]) + new ProcessDsl(config).applyConfigSelectorWithName(settings, 'delta') + then: + config.cpus == 2 + config.disk == '100GB' + config.size() == 2 + + when: + config = new ProcessConfig([:]) + new ProcessDsl(config).applyConfigSelectorWithName(settings, 'gamma') + then: + config.disk == '100GB' + config.size() == 1 + + when: + config = new ProcessConfig([:]) + new ProcessDsl(config).applyConfigSelectorWithName(settings, 'omega_x') + then: + config.cpus == 4 + config.size() == 1 + } + + + def 'should apply config process defaults' () { + + when: + def builder = new ProcessDsl(Mock(BaseScript), null) + builder.queue 'cn-el6' + builder.memory '10 GB' + builder.applyConfigDefaults( + queue: 'def-queue', + container: 'ubuntu:latest' + ) + def config = builder.getConfig() + + then: + config.queue == 'cn-el6' + config.container == 'ubuntu:latest' + config.memory == '10 GB' + config.cacheable == true + + + + when: + builder = new ProcessDsl(Mock(BaseScript), null) + builder.container null + builder.applyConfigDefaults( + queue: 'def-queue', + container: 'ubuntu:latest', + maxRetries: 5 + ) + config = builder.getConfig() + then: + config.queue == 'def-queue' + config.container == null + config.maxRetries == 5 + + + + when: + builder = new ProcessDsl(Mock(BaseScript), null) + builder.maxRetries 10 + builder.applyConfigDefaults( + queue: 'def-queue', + container: 'ubuntu:latest', + maxRetries: 5 + ) + config = builder.getConfig() + then: + config.queue == 'def-queue' + config.container == 'ubuntu:latest' + config.maxRetries == 10 + } + + + def 'should apply pod configs' () { + + when: + def builder = new ProcessDsl(Mock(BaseScript), null) + builder.applyConfigDefaults( pod: [secret: 'foo', mountPath: '/there'] ) + then: + builder.getConfig().pod == [ + [secret: 'foo', mountPath: '/there'] + ] + + when: + builder = new ProcessDsl(Mock(BaseScript), null) + builder.applyConfigDefaults( pod: [ + [secret: 'foo', mountPath: '/here'], + [secret: 'bar', mountPath: '/there'] + ] ) + then: + builder.getConfig().pod == [ + [secret: 'foo', mountPath: '/here'], + [secret: 'bar', mountPath: '/there'] + ] + + } + + def 'should clone config object' () { + + when: + def builder = new ProcessDsl(Mock(BaseScript), null) + def config = builder.getConfig() + builder.queue 'cn-el6' + builder.container 'ubuntu:latest' + builder.memory '10 GB' + builder._in_val('foo') + builder._in_file('sample.txt') + builder._out_file('result.txt') + + then: + config.queue == 'cn-el6' + config.container == 'ubuntu:latest' + config.memory == '10 GB' + config.getInputs().size() == 2 + config.getOutputs().size() == 1 + + when: + def copy = config.clone() + builder = new ProcessDsl(copy) + builder.queue 'long' + builder.container 'debian:wheezy' + builder.memory '5 GB' + builder._in_val('bar') + builder._out_file('sample.bam') + + then: + copy.queue == 'long' + copy.container == 'debian:wheezy' + copy.memory == '5 GB' + copy.getInputs().size() == 3 + copy.getOutputs().size() == 2 + + // original config is not affected + config.queue == 'cn-el6' + config.container == 'ubuntu:latest' + config.memory == '10 GB' + config.getInputs().size() == 2 + config.getOutputs().size() == 1 + } + + def 'should apply accelerator config' () { + + given: + def builder = new ProcessDsl(Mock(BaseScript), null) + def config = builder.getConfig() + + when: + builder.accelerator 5 + then: + config.accelerator == [limit: 5] + + when: + builder.accelerator request: 1, limit: 5, type: 'nvida' + then: + config.accelerator == [request: 1, limit: 5, type: 'nvida'] + + when: + builder.accelerator 5, type: 'nvida' + then: + config.accelerator == [limit: 5, type: 'nvida'] + + when: + builder.accelerator 1, limit: 5 + then: + config.accelerator == [request: 1, limit:5] + + when: + builder.accelerator 5, request: 1 + then: + config.accelerator == [request: 1, limit:5] + } + + def 'should apply disk config' () { + + given: + def builder = new ProcessDsl(Mock(BaseScript), null) + def config = builder.getConfig() + + when: + builder.disk '100 GB' + then: + config.disk == [request: '100 GB'] + + when: + builder.disk '375 GB', type: 'local-ssd' + then: + config.disk == [request: '375 GB', type: 'local-ssd'] + + when: + builder.disk request: '375 GB', type: 'local-ssd' + then: + config.disk == [request: '375 GB', type: 'local-ssd'] + } + + def 'should apply architecture config' () { + + given: + def builder = new ProcessDsl(Mock(BaseScript), null) + def config = builder.getConfig() + + when: + builder.arch 'linux/x86_64' + then: + config.arch == [name: 'linux/x86_64'] + + when: + builder.arch 'linux/x86_64', target: 'zen3' + then: + config.arch == [name: 'linux/x86_64', target: 'zen3'] + + when: + builder.arch name: 'linux/x86_64', target: 'zen3' + then: + config.arch == [name: 'linux/x86_64', target: 'zen3'] + } + + def 'should not apply config on negative label' () { + given: + def settings = [ + 'withLabel:foo': [ cpus: 2 ], + 'withLabel:!foo': [ cpus: 4 ], + 'withLabel:!nodisk_.*': [ disk: '100.GB'] + ] + + when: + def config = new ProcessConfig(label: ['foo', 'other']) + new ProcessDsl(config).applyConfig(settings, "processName", null, null) + then: + config.cpus == 2 + config.disk == '100.GB' + + when: + config = new ProcessConfig(label: ['foo', 'other', 'nodisk_label']) + new ProcessDsl(config).applyConfig(settings, "processName", null, null) + then: + config.cpus == 2 + !config.disk + + when: + config = new ProcessConfig(label: ['other', 'nodisk_label']) + new ProcessDsl(config).applyConfig(settings, "processName", null, null) + then: + config.cpus == 4 + !config.disk + + } + + def 'should throw exception for invalid error strategy' () { + when: + def builder = new ProcessDsl(Mock(BaseScript), null) + builder.errorStrategy 'abort' + + then: + def e = thrown(IllegalArgumentException) + e.message == "Unknown error strategy 'abort' ― Available strategies are: terminate,finish,ignore,retry" + + } + + def 'should not throw exception for valid error strategy or closure' () { + when: + def builder = new ProcessDsl(Mock(BaseScript), null) + builder.errorStrategy 'retry' + + then: + noExceptionThrown() + + when: + builder = new ProcessDsl(Mock(BaseScript), null) + builder.errorStrategy 'terminate' + + then: + noExceptionThrown() + + when: + builder = new ProcessDsl(Mock(BaseScript), null) + builder.errorStrategy { task.exitStatus==14 ? 'retry' : 'terminate' } + + then: + noExceptionThrown() + } +} diff --git a/modules/nextflow/src/test/groovy/nextflow/script/params/ParamsDsl2Test.groovy b/modules/nextflow/src/test/groovy/nextflow/script/params/ParamsDsl2Test.groovy index b703b22187..a710fefd5b 100644 --- a/modules/nextflow/src/test/groovy/nextflow/script/params/ParamsDsl2Test.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/script/params/ParamsDsl2Test.groovy @@ -1,7 +1,7 @@ package nextflow.script.params import nextflow.Session -import nextflow.ast.NextflowDSL +import nextflow.ast.NextflowXform import nextflow.script.BaseScript import nextflow.script.ScriptBinding import nextflow.script.ScriptMeta @@ -114,7 +114,7 @@ class ParamsDsl2Test extends Dsl2Spec { and: def config = new CompilerConfiguration() config.setScriptBaseClass(BaseScript.class.name) - config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowDSL)) + config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowXform)) def SCRIPT = ''' @@ -155,7 +155,7 @@ class ParamsDsl2Test extends Dsl2Spec { and: def config = new CompilerConfiguration() config.setScriptBaseClass(BaseScript.class.name) - config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowDSL)) + config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowXform)) def SCRIPT = ''' diff --git a/modules/nextflow/src/test/groovy/nextflow/script/params/ParamsInTest.groovy b/modules/nextflow/src/test/groovy/nextflow/script/params/ParamsInTest.groovy index 3312a37a3c..7e5da2b208 100644 --- a/modules/nextflow/src/test/groovy/nextflow/script/params/ParamsInTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/script/params/ParamsInTest.groovy @@ -631,7 +631,7 @@ class ParamsInTest extends Dsl2Spec { in0.inChannel instanceof DataflowVariable in0.inChannel.val == ['aaa'] in0.inner.name == 'x' - in0.inner.owner == in0 + in0.inner.owner.toString() == in0.toString() in1.class == EachInParam in1.name == '__$eachinparam<1>' @@ -639,14 +639,14 @@ class ParamsInTest extends Dsl2Spec { in1.inChannel.val == [1,2] in1.inner.name == 'p' in1.inner instanceof ValueInParam - in1.inner.owner == in1 + in1.inner.owner.toString() == in1.toString() in2.class == EachInParam in2.name == '__$eachinparam<2>' in2.inChannel.val == [1,2,3] in2.inner instanceof ValueInParam in2.inner.name == 'z' - in2.inner.owner == in2 + in2.inner.owner.toString() == in2.toString() in3.class == EachInParam in3.name == '__$eachinparam<3>' @@ -654,7 +654,7 @@ class ParamsInTest extends Dsl2Spec { in3.inChannel.val == ['file-a.txt'] in3.inner instanceof FileInParam in3.inner.name == 'foo' - in3.inner.owner == in3 + in3.inner.owner.toString() == in3.toString() in4.class == EachInParam in4.name == '__$eachinparam<4>' @@ -663,7 +663,7 @@ class ParamsInTest extends Dsl2Spec { in4.inner instanceof FileInParam in4.inner.name == 'bar' in4.inner.filePattern == 'bar' - in4.inner.owner == in4 + in4.inner.owner.toString() == in4.toString() } @@ -940,19 +940,19 @@ class ParamsInTest extends Dsl2Spec { in0.inChannel instanceof DataflowVariable in0.inChannel.val == ['file-a.txt'] in0.inner instanceof FileInParam - (in0.inner as FileInParam).name == 'foo' - (in0.inner as FileInParam).owner == in0 - (in0.inner as FileInParam).isPathQualifier() + in0.inner.name == 'foo' + in0.inner.owner.toString() == in0.toString() + in0.inner.isPathQualifier() in1.class == EachInParam in1.name == '__$eachinparam<1>' in1.inChannel instanceof DataflowVariable in1.inChannel.val == ['file-x.fa'] in1.inner instanceof FileInParam - (in1.inner as FileInParam).name == 'bar' - (in1.inner as FileInParam).filePattern == 'bar' - (in1.inner as FileInParam).owner == in1 - (in1.inner as FileInParam).isPathQualifier() + in1.inner.name == 'bar' + in1.inner.filePattern == 'bar' + in1.inner.owner.toString() == in1.toString() + in1.inner.isPathQualifier() } From 5ad981301c154cc0e890ab26f5b95617152382f9 Mon Sep 17 00:00:00 2001 From: Ben Sherman Date: Wed, 29 Nov 2023 20:15:08 -0600 Subject: [PATCH 03/36] Add ProcessFn annotation Signed-off-by: Ben Sherman --- docs/operator.md | 7 + .../nextflow/ast/NextflowXformImpl.groovy | 1 + .../main/groovy/nextflow/ast/ProcessFn.groovy | 42 ++++ .../groovy/nextflow/ast/ProcessFnXform.groovy | 208 ++++++++++++++++++ .../extension/DefaultMergeClosure.groovy | 26 ++- .../groovy/nextflow/extension/MergeOp.groovy | 12 +- .../nextflow/extension/OperatorImpl.groovy | 18 +- .../nextflow/processor/TaskProcessor.groovy | 71 +++--- .../groovy/nextflow/processor/TaskRun.groovy | 10 +- .../groovy/nextflow/script/BaseScript.groovy | 69 ++++-- .../nextflow/script/ProcessConfig.groovy | 19 ++ .../groovy/nextflow/script/ProcessDef.groovy | 42 +++- .../groovy/nextflow/script/ScriptMeta.groovy | 2 + .../nextflow/script/ScriptParser.groovy | 2 + .../nextflow/script/params/BaseInParam.groovy | 4 + .../src/main/nextflow/io/ValueObject.groovy | 2 +- 16 files changed, 453 insertions(+), 82 deletions(-) create mode 100644 modules/nextflow/src/main/groovy/nextflow/ast/ProcessFn.groovy create mode 100644 modules/nextflow/src/main/groovy/nextflow/ast/ProcessFnXform.groovy diff --git a/docs/operator.md b/docs/operator.md index a54f0b022e..9593fcc248 100644 --- a/docs/operator.md +++ b/docs/operator.md @@ -847,6 +847,13 @@ An optional closure can be provided to customise the items emitted by the result :language: console ``` +Available options: + +`flat` +: :::{versionadded} 24.10.0 + ::: +: When `true`, automatically flattens merged items by one level (default: `true`). This option is ignored when a mapping closure is specified. + :::{danger} In general, the use of the `merge` operator is discouraged. Processes and channel operators are not guaranteed to emit items in the order that they were received, as they are executed concurrently. Therefore, if you try to merge output channels from different processes, the resulting channel may be different on each run, which will cause resumed runs to {ref}`not work properly `. diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/NextflowXformImpl.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/NextflowXformImpl.groovy index fb88728ae6..f5dfe8abed 100644 --- a/modules/nextflow/src/main/groovy/nextflow/ast/NextflowXformImpl.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/ast/NextflowXformImpl.groovy @@ -47,6 +47,7 @@ class NextflowXformImpl implements ASTTransformation { new BinaryExpressionXform(unit).visitClass(classNode) new DslCodeVisitor(unit).visitClass(classNode) new OperatorXform(unit).visitClass(classNode) + new ProcessFnXform(unit).visitClass(classNode) } } diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/ProcessFn.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/ProcessFn.groovy new file mode 100644 index 0000000000..c00e45c8fc --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/ast/ProcessFn.groovy @@ -0,0 +1,42 @@ +/* + * Copyright 2013-2023, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.ast + +import java.lang.annotation.ElementType +import java.lang.annotation.Retention +import java.lang.annotation.RetentionPolicy +import java.lang.annotation.Target + +/** + * Annotation for process functions. + * + * @author Ben Sherman + */ +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.METHOD) +@interface ProcessFn { + Class directives() + Class inputs() + Class outputs() + + boolean script() default false + boolean shell() default false + + // injected via AST transform + Class params() + String source() +} diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/ProcessFnXform.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/ProcessFnXform.groovy new file mode 100644 index 0000000000..70df7b9397 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/ast/ProcessFnXform.groovy @@ -0,0 +1,208 @@ +/* + * Copyright 2013-2023, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.ast + +import static org.codehaus.groovy.ast.tools.GeneralUtils.* + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import nextflow.processor.TaskConfig +import org.codehaus.groovy.ast.ASTNode +import org.codehaus.groovy.ast.ClassCodeVisitorSupport +import org.codehaus.groovy.ast.ClassNode +import org.codehaus.groovy.ast.MethodNode +import org.codehaus.groovy.ast.Parameter +import org.codehaus.groovy.ast.expr.ArgumentListExpression +import org.codehaus.groovy.ast.expr.BinaryExpression +import org.codehaus.groovy.ast.expr.ClosureExpression +import org.codehaus.groovy.ast.expr.Expression +import org.codehaus.groovy.ast.expr.ListExpression +import org.codehaus.groovy.ast.expr.MethodCallExpression +import org.codehaus.groovy.ast.expr.UnaryMinusExpression +import org.codehaus.groovy.ast.expr.VariableExpression +import org.codehaus.groovy.ast.stmt.BlockStatement +import org.codehaus.groovy.ast.stmt.ExpressionStatement +import org.codehaus.groovy.ast.stmt.Statement +import org.codehaus.groovy.control.SourceUnit +import org.codehaus.groovy.syntax.SyntaxException +import org.codehaus.groovy.syntax.Types +/** + * Implements syntax transformations for process functions. + * + * @author Ben Sherman + */ +@Slf4j +@CompileStatic +class ProcessFnXform extends ClassCodeVisitorSupport { + + private final SourceUnit unit + + ProcessFnXform(SourceUnit unit) { + this.unit = unit + } + + @Override + protected SourceUnit getSourceUnit() { unit } + + @Override + void visitMethod(MethodNode method) { + final annotation = method.getAnnotations() + .find(a -> a.getClassNode().getName() == 'ProcessFn') + + if( annotation ) + transform(method, annotation.getMembers()) + } + + /** + * Transform a ProcessFn annotation to be semantically valid. + * + * @param method + * @param opts + */ + protected void transform(MethodNode method, Map opts) { + // fix directives + final directives = opts['directives'] + if( directives != null && directives instanceof ClosureExpression ) { + final block = (BlockStatement)directives.getCode() + for( Statement stmt : block.getStatements() ) + fixDirectiveWithNegativeValue(stmt) + } + + // fix inputs + final inputs = opts['inputs'] + if( inputs != null && inputs instanceof ClosureExpression ) { + final block = (BlockStatement)inputs.getCode() + for( Statement stmt : block.getStatements() ) + fixInputMethod((ExpressionStatement)stmt) + } + + // fix outputs + final outputs = opts['outputs'] + if( outputs != null && outputs instanceof ClosureExpression ) { + final block = (BlockStatement)outputs.getCode() + for( Statement stmt : block.getStatements() ) + fixOutputMethod((ExpressionStatement)stmt) + } + + // insert `task` method parameter + final params = method.getParameters() as List + params.push(new Parameter(new ClassNode(TaskConfig), 'task')) + method.setParameters(params as Parameter[]) + + // TODO: append stub source + + // append method params + opts.put( 'params', closureX( block( new ExpressionStatement( + new ListExpression( + params.collect(p -> (Expression)constX(p.getName())) + ) + ) ) ) ) + + // append script source + opts.put( 'source', constX( getSource(method.getCode()) ) ) + } + + /** + * Fix directives with a single argument with a negative value, + * since it will be parsed as a subtract expression if there are + * no parentheses. + * + * @param stmt + */ + protected void fixDirectiveWithNegativeValue(Statement stmt) { + // -- check for binary subtract expression + if( stmt !instanceof ExpressionStatement ) + return + def expr = ((ExpressionStatement)stmt).getExpression() + if( expr !instanceof BinaryExpression ) + return + def binary = (BinaryExpression)expr + if( binary.leftExpression !instanceof VariableExpression ) + return + if( binary.operation.type != Types.MINUS ) + return + + // -- transform binary expression `NAME - ARG` into method call `NAME(-ARG)` + def name = ((VariableExpression)binary.leftExpression).name + def arg = (Expression)new UnaryMinusExpression(binary.rightExpression) + + ((ExpressionStatement)stmt).setExpression( new MethodCallExpression( + VariableExpression.THIS_EXPRESSION, + name, + new ArgumentListExpression(arg) + ) ) + } + + private static final VALID_INPUT_METHODS = ['env','file','path','stdin'] + + /** + * Fix input method calls. + * + * @param stmt + */ + protected void fixInputMethod(ExpressionStatement stmt) { + final methodCall = (MethodCallExpression)stmt.getExpression() + final name = methodCall.getMethodAsString() + final args = (ArgumentListExpression)methodCall.getArguments() + + if( name !in VALID_INPUT_METHODS ) + syntaxError(stmt, "Invalid input method '${name}'") + + methodCall.setMethod( constX('_in_' + name) ) + } + + private static final VALID_OUTPUT_METHODS = ['val','env','file','path','stdout','tuple'] + + /** + * Fix output method calls. + * + * @param stmt + */ + protected void fixOutputMethod(ExpressionStatement stmt) { + final methodCall = (MethodCallExpression)stmt.getExpression() + final name = methodCall.getMethodAsString() + final args = (ArgumentListExpression)methodCall.getArguments() + + if( name !in VALID_OUTPUT_METHODS ) + syntaxError(stmt, "Invalid output method '${name}'") + + methodCall.setMethod( constX('_out_' + name) ) + } + + private String getSource(ASTNode node) { + final buffer = new StringBuilder() + final colx = node.getColumnNumber() + final colz = node.getLastColumnNumber() + final first = node.getLineNumber() + final last = node.getLastLineNumber() + for( int i=first; i<=last; i++ ) { + def line = unit.source.getLine(i, null) + if( i==last ) + line = line.substring(0,colz-1) + if( i==first ) + line = line.substring(colx-1) + buffer.append(line) .append('\n') + } + + return buffer.toString() + } + + protected void syntaxError(ASTNode node, String message) { + unit.addError( new SyntaxException(message, node.lineNumber, node.columnNumber) ) + } + +} diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/DefaultMergeClosure.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/DefaultMergeClosure.groovy index e904834b89..99062f3518 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/DefaultMergeClosure.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/DefaultMergeClosure.groovy @@ -25,9 +25,12 @@ class DefaultMergeClosure extends Closure { private int numOfParams - DefaultMergeClosure(int n) { - super(null, null); - numOfParams = n + private boolean flat + + DefaultMergeClosure(int n, boolean flat) { + super(null, null) + this.numOfParams = n + this.flat = flat } @Override @@ -42,12 +45,12 @@ class DefaultMergeClosure extends Closure { @Override public void setDelegate(final Object delegate) { - super.setDelegate(delegate); + super.setDelegate(delegate) } @Override public void setResolveStrategy(final int resolveStrategy) { - super.setResolveStrategy(resolveStrategy); + super.setResolveStrategy(resolveStrategy) } @Override @@ -58,10 +61,15 @@ class DefaultMergeClosure extends Closure { @Override public Object call(final Object... args) { final result = [] - for( int i=0; i others + private boolean flat private Closure closure - MergeOp(final DataflowReadChannel source, final List others, final Closure closure=null) { + MergeOp(DataflowReadChannel source, List others, Map opts=null, Closure closure=null) { this.source = source this.others = others + this.flat = opts?.flat!=null ? opts?.flat : true this.closure = closure } - MergeOp(final DataflowReadChannel source, final DataflowReadChannel other, final Closure closure=null ) { - this.source = source - this.others = Collections.singletonList(other) - this.closure = closure + MergeOp(DataflowReadChannel source, DataflowReadChannel other, Map opts=null, Closure closure=null) { + this(source, Collections.singletonList(other), opts, closure) } DataflowWriteChannel apply() { final result = CH.createBy(source) final List inputs = new ArrayList(1 + others.size()) - final action = closure ? new ChainWithClosure<>(closure) : new DefaultMergeClosure(1 + others.size()) + final action = closure ? new ChainWithClosure<>(closure) : new DefaultMergeClosure(1 + others.size(), flat) inputs.add(source) inputs.addAll(others) final listener = stopErrorListener(source,result) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy index ceeb26f858..2e2924bf6f 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy @@ -1077,24 +1077,18 @@ class OperatorImpl { } // NO DAG - DataflowWriteChannel merge(final DataflowReadChannel source, final DataflowReadChannel other, final Closure closure=null) { - final result = CH.createBy(source) - final inputs = [source, other] - final action = closure ? new ChainWithClosure<>(closure) : new DefaultMergeClosure(inputs.size()) - final listener = stopErrorListener(source,result) - final params = createOpParams(inputs, result, listener) - newOperator(params, action) - return result; + DataflowWriteChannel merge(DataflowReadChannel source, Map opts=null, DataflowReadChannel other, Closure closure=null) { + new MergeOp(source, other, opts, closure).apply() } // NO DAG - DataflowWriteChannel merge(final DataflowReadChannel source, final DataflowReadChannel... others) { - new MergeOp(source,others as List).apply() + DataflowWriteChannel merge(DataflowReadChannel source, Map opts=null, DataflowReadChannel... others) { + new MergeOp(source, others as List, opts).apply() } // NO DAG - DataflowWriteChannel merge(final DataflowReadChannel source, final List others, final Closure closure=null) { - new MergeOp(source,others,closure).apply() + DataflowWriteChannel merge(DataflowReadChannel source, Map opts=null, List others, Closure closure=null) { + new MergeOp(source, others, opts, closure).apply() } DataflowWriteChannel randomSample(DataflowReadChannel source, int n, Long seed = null) { diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy index 5b8dfa2774..495ecf0f63 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy @@ -66,6 +66,7 @@ import nextflow.exception.ProcessFailedException import nextflow.exception.ProcessRetryableException import nextflow.exception.ProcessSubmitTimeoutException import nextflow.exception.ProcessUnrecoverableException +import nextflow.exception.ScriptRuntimeException import nextflow.exception.ShowOnlyExceptionMessage import nextflow.exception.UnexpectedException import nextflow.executor.CachedTaskHandler @@ -396,23 +397,7 @@ class TaskProcessor { log.warn(msg) } - /** - * Launch the 'script' define by the code closure as a local bash script - * - * @param code A {@code Closure} returning a bash script e.g. - * 
    -     *              {
-     *                 """
-     *                 #!/bin/bash
-     *                 do this ${x}
-     *                 do that ${y}
-     *                 :
-     *                 """
-     *              }
-     *
-     * @return {@code this} instance
-     */
-    def run() {
+    def run(DataflowReadChannel source) {
 
         // -- check that the task has a body
         if ( !taskBody )
@@ -468,7 +453,7 @@ class TaskProcessor {
         session.processRegister(this)
 
         // create the underlying dataflow operator
-        createOperator()
+        createOperator(source)
 
         session.notifyProcessCreate(this)
 
@@ -480,15 +465,11 @@ class TaskProcessor {
         return result.size() == 1 ? result[0] : result
     }
 
-    /**
-     * Template method which extending classes have to override in order to
-     * create the underlying *dataflow* operator associated with this processor
-     *
-     * See {@code DataflowProcessor}
-     */
-
-    protected void createOperator() {
-        def opInputs = new ArrayList(config.getInputs().getChannels())
+    protected void createOperator(DataflowReadChannel source) {
+        def control = config.getInputs().last().getInChannel()
+        def opInputs = source != null
+            ? [source, control]
+            : [control]
 
         /*
          * check if there are some iterators declaration
@@ -599,7 +580,7 @@ class TaskProcessor {
     final protected void invokeTask( Object[] args ) {
         assert args.size()==2
         final params = (TaskStartParams) args[0]
-        final values = (List) args[1]
+        final values = ((List) args[1]).first()
 
         // create and initialize the task instance to be executed
         log.trace "Invoking task > $name with params=$params; values=$values"
@@ -609,6 +590,24 @@ class TaskProcessor {
         // -- set the task instance as the current in this thread
         currentTask.set(task)
 
+        // -- add task config to arguments
+        values.push(task.config)
+
+        // -- validate task arguments
+        if( config.params.size() != values.size() )
+            throw new ScriptRuntimeException("Process $name expected ${config.params.size()} arguments but received ${values.size()}: ${values}")
+
+        for( int i = 0; i < config.params.size(); i++ ) {
+            final param = config.params[i]
+            final value = values[i]
+            if( !param.type.isAssignableFrom(value.class) )
+                log.warn1 "Process $name > expected type ${param.type.name} for param ${param.name} but got a ${value.class.name}"
+        }
+
+        // -- add arguments to task context
+        for( int i = 1; i < config.params.size(); i++ )
+            task.context.put(config.params[i].name, values[i])
+
         // -- validate input lengths
         validateInputTuples(values)
 
@@ -627,7 +626,7 @@ class TaskProcessor {
         }
         else {
             // -- resolve the task command script
-            task.resolve(taskBody)
+            task.resolve(taskBody, config.params*.name)
         }
 
         // -- verify if exists a stored result for this case,
@@ -2018,7 +2017,19 @@ class TaskProcessor {
         task.inputs.keySet().each { InParam param ->
 
             // add the value to the task instance
-            def val = param.decodeInputs(values)
+            def bindObject = param.getBindObject()
+
+            def val
+            if( bindObject instanceof Closure ) {
+                final cl = (Closure)bindObject.clone()
+                cl.delegate = task.context
+                cl.resolveStrategy = Closure.DELEGATE_FIRST
+                val = cl.call()
+            }
+            else
+                val = bindObject
+
+            log.trace "Process $name > binding param ${param.class.name} to ${val}"
 
             switch(param) {
                 case ValueInParam:
diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy
index 205951016e..ec2842cecf 100644
--- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy
@@ -710,9 +710,10 @@ class TaskRun implements Cloneable {
      * 2) extract the process code `source`
      * 3) assign the `script` code to execute
      *
-     * @param body A {@code BodyDef} object instance
+     * @param body
+     * @param params
      */
-    @PackageScope void resolve(BodyDef body) {
+    @PackageScope void resolve(BodyDef body, List params=[]) {
 
         // -- initialize the task code to be executed
         this.code = body.closure.clone() as Closure
@@ -727,11 +728,14 @@ class TaskRun implements Cloneable {
         if( body.type != ScriptType.SCRIPTLET )
             return
 
+        // collect method args from task context
+        final args = params.collect(param -> context[param])
+
         // Important!
         // when the task is implemented by a script string
         // Invoke the closure which returns the script with all the variables replaced with the actual values
         try {
-            final result = code.call()
+            final result = code.call(*args)
             if ( result instanceof Path ) {
                 script = renderTemplate(result, body.isShell)
             }
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy b/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy
index e70a01c49c..c8c4a846a5 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy
@@ -19,9 +19,11 @@ package nextflow.script
 import java.lang.reflect.InvocationTargetException
 import java.nio.file.Paths
 
+import groovy.transform.CompileStatic
 import groovy.util.logging.Slf4j
 import nextflow.NextflowMeta
 import nextflow.Session
+import nextflow.ast.ProcessFn
 import nextflow.exception.AbortOperationException
 import nextflow.script.dsl.ProcessDsl
 import nextflow.script.dsl.WorkflowDsl
@@ -144,22 +146,63 @@ abstract class BaseScript extends Script implements ExecutionContext {
         include .setSession(session)
     }
 
-    /**
-     * Invokes custom methods in the task execution context
-     *
-     * @see nextflow.processor.TaskContext#invokeMethod(java.lang.String, java.lang.Object)
-     * @see WorkflowBinding#invokeMethod(java.lang.String, java.lang.Object)
-     *
-     * @param name the name of the method to call
-     * @param args the arguments to use for the method call
-     * @return The result of the custom method execution
-     */
-    @Override
-    Object invokeMethod(String name, Object args) {
-        binding.invokeMethod(name, args)
+    @CompileStatic
+    private void registerProcessFunctions() {
+        final dslEval = { Object delegate, Class clazz ->
+            final cl = clazz.newInstance(this, this)
+            cl.delegate = delegate
+            cl.resolveStrategy = Closure.DELEGATE_FIRST
+            cl.call()
+        }
+
+        final clazz = this.getClass()
+        for( final method : clazz.getDeclaredMethods() ) {
+            // check for ProcessFn annotation
+            final name = method.getName()
+            final processFn = method.getAnnotation(ProcessFn)
+            if( !processFn )
+                continue
+
+            // validate annotation
+            if( processFn.script() && processFn.shell() )
+                throw new IllegalArgumentException("Process function `${name}` cannot have script and shell enabled simultaneously")
+
+            // build process from annotation
+            final builder = new ProcessDsl(this, name)
+            dslEval(builder, processFn.directives())
+            dslEval(builder, processFn.inputs())
+            dslEval(builder, processFn.outputs())
+
+            // get method parameters
+            final paramNames = (List)((Closure)processFn.params().newInstance(this, this)).call()
+            final params = (0 ..< paramNames.size()).collect( i ->
+                new Parameter( paramNames[i], method.getParameters()[i].getType() )
+            )
+
+            builder.config.params = params
+
+            // determine process type
+            def type
+            if( processFn.script() )
+                type = 'script'
+            else if( processFn.shell() )
+                type = 'shell'
+            else
+                type = 'exec'
+
+            // create task body
+            final taskBody = new BodyDef( this.&"${name}", processFn.source(), type, [] )
+
+            final process = builder.withBody(taskBody).build()
+            meta.addDefinition(process)
+        }
     }
 
     private run0() {
+        // register any process functions
+        registerProcessFunctions()
+
+        // execute script
         final result = runScript()
         if( meta.isModule() ) {
             return result
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessConfig.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessConfig.groovy
index fcd0b132fa..ffdaa173c8 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessConfig.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessConfig.groovy
@@ -16,6 +16,7 @@
 
 package nextflow.script
 
+import groovy.transform.Immutable
 import groovy.transform.PackageScope
 import groovy.util.logging.Slf4j
 import nextflow.Const
@@ -66,6 +67,11 @@ class ProcessConfig implements Map, Cloneable {
      */
     private String processName
 
+    /**
+     * List of parameters defined by a process function.
+     */
+    List params
+
     /**
      * List of process input definitions
      */
@@ -118,6 +124,9 @@ class ProcessConfig implements Map, Cloneable {
     Object getProperty( String name ) {
 
         switch( name ) {
+            case 'params':
+                return getParams()
+
             case 'inputs':
                 return getInputs()
 
@@ -152,6 +161,10 @@ class ProcessConfig implements Map, Cloneable {
         return new TaskConfig(configProperties)
     }
 
+    List getParams() {
+        params
+    }
+
     InputsList getInputs() {
         inputs
     }
@@ -216,3 +229,9 @@ class ProcessConfig implements Map, Cloneable {
     }
 
 }
+
+@Immutable
+class Parameter {
+    String name
+    Class type
+}
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy
index b308641415..b3d80d4ae8 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy
@@ -18,11 +18,14 @@ package nextflow.script
 
 import groovy.transform.CompileStatic
 import groovy.util.logging.Slf4j
+import groovyx.gpars.dataflow.DataflowBroadcast
+import groovyx.gpars.dataflow.DataflowReadChannel
 import nextflow.Const
 import nextflow.Global
 import nextflow.Session
 import nextflow.exception.ScriptRuntimeException
 import nextflow.extension.CH
+import nextflow.extension.MergeOp
 import nextflow.script.dsl.ProcessDsl
 import nextflow.script.params.BaseInParam
 import nextflow.script.params.BaseOutParam
@@ -148,16 +151,17 @@ class ProcessDef extends BindableDef implements IterableDef, ChainableDef {
         // initialise process config
         initialize()
 
-        // get params 
-        final params = ChannelOut.spread(args)
-        // sanity check
-        if( params.size() != declaredInputs.size() )
-            throw new ScriptRuntimeException(missMatchErrMessage(processName, declaredInputs.size(), params.size()))
+        // create merged input channel
+        final inputs = ChannelOut.spread(args).collect(ch -> getInChannel(ch))
+        if( inputs.findAll(ch -> CH.isChannelQueue(ch)).size() > 1 )
+            throw new ScriptRuntimeException("Process `$name` received multiple queue channel inputs which is not allowed")
+
+        final input = CH.getReadChannel(new MergeOp(inputs.first(), inputs[1..0, "Process output should contains at least one channel"
         return output = new ChannelOut(copyOuts)
     }
 
+    private DataflowReadChannel getInChannel(Object obj) {
+        if( obj == null )
+            throw new IllegalArgumentException('A process input channel evaluates to null')
+
+        final result = obj instanceof Closure
+            ? obj.call()
+            : obj
+
+        if( result == null )
+            throw new IllegalArgumentException('A process input channel evaluates to null')
+
+        def inChannel
+        if ( result instanceof DataflowReadChannel || result instanceof DataflowBroadcast )
+            inChannel = CH.getReadChannel(result)
+        else {
+            inChannel = CH.value()
+            inChannel.bind(result)
+        }
+
+        return inChannel
+    }
+
 }
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ScriptMeta.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ScriptMeta.groovy
index 784b40d784..93a40ee2e6 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ScriptMeta.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ScriptMeta.groovy
@@ -25,6 +25,7 @@ import groovy.transform.Memoized
 import groovy.transform.PackageScope
 import groovy.util.logging.Slf4j
 import nextflow.NF
+import nextflow.ast.ProcessFn
 import nextflow.exception.DuplicateModuleFunctionException
 import nextflow.exception.MissingModuleComponentException
 import nextflow.script.bundle.ResourcesBundle
@@ -175,6 +176,7 @@ class ScriptMeta {
             if( Modifier.isStatic(method.getModifiers())) continue
             if( method.name.startsWith('super$')) continue
             if( method.name in INVALID_FUNCTION_NAMES ) continue
+            if( method.isAnnotationPresent(ProcessFn) ) continue
 
             // If method is already into the list, maybe with other signature, it's not necessary to include it again
             if( result.find{it.name == method.name}) continue
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ScriptParser.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ScriptParser.groovy
index 28af74c85e..a3801fa65a 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ScriptParser.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ScriptParser.groovy
@@ -24,6 +24,7 @@ import nextflow.Channel
 import nextflow.Nextflow
 import nextflow.Session
 import nextflow.ast.NextflowXform
+import nextflow.ast.ProcessFn
 import nextflow.exception.ScriptCompilationException
 import nextflow.extension.FilesEx
 import nextflow.file.FileHelper
@@ -112,6 +113,7 @@ class ScriptParser {
         importCustomizer.addImports( Channel.name )
         importCustomizer.addImports( Duration.name )
         importCustomizer.addImports( MemoryUnit.name )
+        importCustomizer.addImports( ProcessFn.name )
         importCustomizer.addImports( ValueObject.name )
         importCustomizer.addImport( 'channel', Channel.name )
         importCustomizer.addStaticStars( Nextflow.name )
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/BaseInParam.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/BaseInParam.groovy
index efe613805b..a932af4863 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/params/BaseInParam.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/params/BaseInParam.groovy
@@ -54,6 +54,10 @@ abstract class BaseInParam extends BaseParam implements InParam {
         return inChannel
     }
 
+    def getBindObject() {
+        bindObject
+    }
+
     BaseInParam( ProcessConfig config ) {
         this(config.getOwnerScript().getBinding(), config.getInputs())
     }
diff --git a/modules/nf-commons/src/main/nextflow/io/ValueObject.groovy b/modules/nf-commons/src/main/nextflow/io/ValueObject.groovy
index 620fcb1dc9..0004e25ab9 100644
--- a/modules/nf-commons/src/main/nextflow/io/ValueObject.groovy
+++ b/modules/nf-commons/src/main/nextflow/io/ValueObject.groovy
@@ -32,7 +32,7 @@ import groovy.transform.Immutable
  *  @author Paolo Di Tommaso 
  */
 @AutoClone
-@Immutable(copyWith=true)
+@Immutable(copyWith=true, knownImmutableClasses=[java.nio.file.Path])
 @SerializableObject
 @AnnotationCollector(mode = AnnotationCollectorMode.PREFER_EXPLICIT_MERGED)
 @Retention(RetentionPolicy.RUNTIME)

From 01ef1dbc6dc05a1f7a4357e7f896514d07430137 Mon Sep 17 00:00:00 2001
From: Ben Sherman 
Date: Wed, 29 Nov 2023 20:38:34 -0600
Subject: [PATCH 04/36] Rename ProcessDsl -> ProcessBuilder, add separate
 builder for process annotation inputs

Signed-off-by: Ben Sherman 
---
 .../src/main/groovy/nextflow/Session.groovy   |  4 +-
 .../groovy/nextflow/ast/ProcessFnXform.groovy | 26 -------
 .../groovy/nextflow/script/BaseScript.groovy  | 11 ++-
 .../groovy/nextflow/script/ProcessDef.groovy  |  4 +-
 .../nextflow/script/ProcessFactory.groovy     |  4 +-
 ...rocessDsl.groovy => ProcessBuilder.groovy} | 10 +--
 .../script/dsl/ProcessInputsBuilder.groovy    | 63 +++++++++++++++
 ...lTest.groovy => ProcessBuilderTest.groovy} | 76 +++++++++----------
 8 files changed, 119 insertions(+), 79 deletions(-)
 rename modules/nextflow/src/main/groovy/nextflow/script/dsl/{ProcessDsl.groovy => ProcessBuilder.groovy} (99%)
 create mode 100644 modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessInputsBuilder.groovy
 rename modules/nextflow/src/test/groovy/nextflow/script/dsl/{ProcessDslTest.groovy => ProcessBuilderTest.groovy} (85%)

diff --git a/modules/nextflow/src/main/groovy/nextflow/Session.groovy b/modules/nextflow/src/main/groovy/nextflow/Session.groovy
index 141c97f789..9572b1c578 100644
--- a/modules/nextflow/src/main/groovy/nextflow/Session.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/Session.groovy
@@ -55,7 +55,7 @@ import nextflow.processor.ErrorStrategy
 import nextflow.processor.TaskFault
 import nextflow.processor.TaskHandler
 import nextflow.processor.TaskProcessor
-import nextflow.script.dsl.ProcessDsl
+import nextflow.script.dsl.ProcessBuilder
 import nextflow.script.BaseScript
 import nextflow.script.ProcessConfig
 import nextflow.script.ProcessFactory
@@ -932,7 +932,7 @@ class Session implements ISession {
      * @return {@code true} if the name specified belongs to the list of process names or {@code false} otherwise
      */
     protected boolean checkValidProcessName(Collection processNames, String selector, List errorMessage)  {
-        final matches = processNames.any { name -> ProcessDsl.matchesSelector(name, selector) }
+        final matches = processNames.any { name -> ProcessBuilder.matchesSelector(name, selector) }
         if( matches )
             return true
 
diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/ProcessFnXform.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/ProcessFnXform.groovy
index 70df7b9397..938c229536 100644
--- a/modules/nextflow/src/main/groovy/nextflow/ast/ProcessFnXform.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/ast/ProcessFnXform.groovy
@@ -82,14 +82,6 @@ class ProcessFnXform extends ClassCodeVisitorSupport {
                 fixDirectiveWithNegativeValue(stmt)
         }
 
-        // fix inputs
-        final inputs = opts['inputs']
-        if( inputs != null && inputs instanceof ClosureExpression ) {
-            final block = (BlockStatement)inputs.getCode()
-            for( Statement stmt : block.getStatements() )
-                fixInputMethod((ExpressionStatement)stmt)
-        }
-
         // fix outputs
         final outputs = opts['outputs']
         if( outputs != null && outputs instanceof ClosureExpression ) {
@@ -147,24 +139,6 @@ class ProcessFnXform extends ClassCodeVisitorSupport {
         ) )
     }
 
-    private static final VALID_INPUT_METHODS = ['env','file','path','stdin']
-
-    /**
-     * Fix input method calls.
-     *
-     * @param stmt
-     */
-    protected void fixInputMethod(ExpressionStatement stmt) {
-        final methodCall = (MethodCallExpression)stmt.getExpression()
-        final name = methodCall.getMethodAsString()
-        final args = (ArgumentListExpression)methodCall.getArguments()
-
-        if( name !in VALID_INPUT_METHODS )
-            syntaxError(stmt, "Invalid input method '${name}'")
-
-        methodCall.setMethod( constX('_in_' + name) )
-    }
-
     private static final VALID_OUTPUT_METHODS = ['val','env','file','path','stdout','tuple']
 
     /**
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy b/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy
index c8c4a846a5..50a1f8b219 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy
@@ -25,7 +25,8 @@ import nextflow.NextflowMeta
 import nextflow.Session
 import nextflow.ast.ProcessFn
 import nextflow.exception.AbortOperationException
-import nextflow.script.dsl.ProcessDsl
+import nextflow.script.dsl.ProcessBuilder
+import nextflow.script.dsl.ProcessInputsBuilder
 import nextflow.script.dsl.WorkflowDsl
 /**
  * Any user defined script will extends this class, it provides the base execution context
@@ -99,7 +100,7 @@ abstract class BaseScript extends Script implements ExecutionContext {
      * @param rawBody
      */
     protected void process(String name, Closure rawBody) {
-        final builder = new ProcessDsl(this, name)
+        final builder = new ProcessBuilder(this, name)
         final copy = (Closure)rawBody.clone()
         copy.delegate = builder
         copy.resolveStrategy = Closure.DELEGATE_FIRST
@@ -168,9 +169,11 @@ abstract class BaseScript extends Script implements ExecutionContext {
                 throw new IllegalArgumentException("Process function `${name}` cannot have script and shell enabled simultaneously")
 
             // build process from annotation
-            final builder = new ProcessDsl(this, name)
+            final builder = new ProcessBuilder(this, name)
+            final inputsBuilder = new ProcessInputsBuilder(builder.getConfig())
+
+            dslEval(inputsBuilder, processFn.inputs())
             dslEval(builder, processFn.directives())
-            dslEval(builder, processFn.inputs())
             dslEval(builder, processFn.outputs())
 
             // get method parameters
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy
index b3d80d4ae8..d68e9625d7 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy
@@ -26,7 +26,7 @@ import nextflow.Session
 import nextflow.exception.ScriptRuntimeException
 import nextflow.extension.CH
 import nextflow.extension.MergeOp
-import nextflow.script.dsl.ProcessDsl
+import nextflow.script.dsl.ProcessBuilder
 import nextflow.script.params.BaseInParam
 import nextflow.script.params.BaseOutParam
 import nextflow.script.params.EachInParam
@@ -96,7 +96,7 @@ class ProcessDef extends BindableDef implements IterableDef, ChainableDef {
 
     protected void initialize() {
         // apply config settings to the process
-        new ProcessDsl(processConfig).applyConfig((Map)session.config.process, baseName, simpleName, processName)
+        new ProcessBuilder(processConfig).applyConfig((Map)session.config.process, baseName, simpleName, processName)
     }
 
     @Override
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessFactory.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessFactory.groovy
index 3d660c8774..b96a6b52ef 100755
--- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessFactory.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessFactory.groovy
@@ -23,7 +23,7 @@ import nextflow.Session
 import nextflow.executor.Executor
 import nextflow.executor.ExecutorFactory
 import nextflow.processor.TaskProcessor
-import nextflow.script.dsl.ProcessDsl
+import nextflow.script.dsl.ProcessBuilder
 /**
  *  Factory class for {@TaskProcessor} instances
  *
@@ -85,7 +85,7 @@ class ProcessFactory {
         assert body
         assert config.process instanceof Map
 
-        final builder = new ProcessDsl(owner, name)
+        final builder = new ProcessBuilder(owner, name)
         // Invoke the code block which will return the script closure to the executed.
         // As side effect will set all the property declarations in the 'taskConfig' object.
         final copy = (Closure)body.clone()
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessDsl.groovy b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessBuilder.groovy
similarity index 99%
rename from modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessDsl.groovy
rename to modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessBuilder.groovy
index f4da53c4db..aded6a3944 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessDsl.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessBuilder.groovy
@@ -33,12 +33,12 @@ import nextflow.script.ProcessConfig
 import nextflow.script.ProcessDef
 
 /**
- * Implements the process DSL.
+ * Implements the process builder DSL.
  *
  * @author Ben Sherman 
  */
 @Slf4j
-class ProcessDsl {
+class ProcessBuilder {
 
     static final List DIRECTIVES = [
             'accelerator',
@@ -88,13 +88,13 @@ class ProcessDsl {
     private BodyDef body
     private ProcessConfig config
 
-    ProcessDsl(BaseScript ownerScript, String processName) {
+    ProcessBuilder(BaseScript ownerScript, String processName) {
         this.ownerScript = ownerScript
         this.processName = processName
         this.config = new ProcessConfig(ownerScript, processName)
     }
 
-    ProcessDsl(ProcessConfig config) {
+    ProcessBuilder(ProcessConfig config) {
         this.ownerScript = config.getOwnerScript()
         this.processName = config.getProcessName()
         this.config = config
@@ -524,7 +524,7 @@ class ProcessDsl {
 
     /// SCRIPT
 
-    ProcessDsl withBody(BodyDef body) {
+    ProcessBuilder withBody(BodyDef body) {
         this.body = body
         return this
     }
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessInputsBuilder.groovy b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessInputsBuilder.groovy
new file mode 100644
index 0000000000..f61bf1483d
--- /dev/null
+++ b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessInputsBuilder.groovy
@@ -0,0 +1,63 @@
+/*
+ * Copyright 2013-2023, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package nextflow.script.dsl
+
+import groovy.util.logging.Slf4j
+import nextflow.script.params.*
+import nextflow.script.ProcessConfig
+
+/**
+ * Process inputs builder DSL for the {@code ProcessFn} annotation.
+ *
+ * @author Ben Sherman 
+ */
+@Slf4j
+class ProcessInputsBuilder {
+
+    private ProcessConfig config
+
+    ProcessInputsBuilder(ProcessConfig config) {
+        this.config = config
+    }
+
+    void env( Object obj ) {
+        new EnvInParam(config).bind(obj)
+    }
+
+    void file( Object obj ) {
+        new FileInParam(config).bind(obj)
+    }
+
+    void path( Map opts=null, Object obj ) {
+        new FileInParam(config)
+                .setPathQualifier(true)
+                .setOptions(opts)
+                .bind(obj)
+    }
+
+    void stdin( Object obj = null ) {
+        def result = new StdInParam(config)
+        if( obj )
+            result.bind(obj)
+        result
+    }
+
+    ProcessConfig getConfig() {
+        return config
+    }
+
+}
diff --git a/modules/nextflow/src/test/groovy/nextflow/script/dsl/ProcessDslTest.groovy b/modules/nextflow/src/test/groovy/nextflow/script/dsl/ProcessBuilderTest.groovy
similarity index 85%
rename from modules/nextflow/src/test/groovy/nextflow/script/dsl/ProcessDslTest.groovy
rename to modules/nextflow/src/test/groovy/nextflow/script/dsl/ProcessBuilderTest.groovy
index 3ba06d411b..198b6150a1 100644
--- a/modules/nextflow/src/test/groovy/nextflow/script/dsl/ProcessDslTest.groovy
+++ b/modules/nextflow/src/test/groovy/nextflow/script/dsl/ProcessBuilderTest.groovy
@@ -33,12 +33,12 @@ import nextflow.util.MemoryUnit
  *
  * @author Paolo Di Tommaso 
  */
-class ProcessDslTest extends Specification {
+class ProcessBuilderTest extends Specification {
 
     def 'should set directives' () {
 
         setup:
-        def builder = new ProcessDsl(Mock(BaseScript), null)
+        def builder = new ProcessBuilder(Mock(BaseScript), null)
         def config = builder.getConfig()
 
         // setting list values
@@ -80,7 +80,7 @@ class ProcessDslTest extends Specification {
     def 'should create input directives' () {
 
         setup:
-        def builder = new ProcessDsl(Mock(BaseScript), null)
+        def builder = new ProcessBuilder(Mock(BaseScript), null)
         def config = builder.getConfig()
 
         when:
@@ -109,7 +109,7 @@ class ProcessDslTest extends Specification {
     def 'should create output directives' () {
 
         setup:
-        def builder = new ProcessDsl(Mock(BaseScript), null)
+        def builder = new ProcessBuilder(Mock(BaseScript), null)
         def config = builder.getConfig()
 
         when:
@@ -133,7 +133,7 @@ class ProcessDslTest extends Specification {
     def 'should create PublishDir object' () {
 
         setup:
-        def builder = new ProcessDsl(Mock(BaseScript), null)
+        def builder = new ProcessBuilder(Mock(BaseScript), null)
         def config = builder.getConfig()
 
         when:
@@ -155,7 +155,7 @@ class ProcessDslTest extends Specification {
     def 'should throw IllegalDirectiveException'() {
 
         given:
-        def builder = new ProcessDsl(Mock(BaseScript), null)
+        def builder = new ProcessBuilder(Mock(BaseScript), null)
 
         when:
         builder.hello 'world'
@@ -174,7 +174,7 @@ class ProcessDslTest extends Specification {
 
     def 'should set process secret'() {
         when:
-        def builder = new ProcessDsl(Mock(BaseScript), null)
+        def builder = new ProcessBuilder(Mock(BaseScript), null)
         def config = builder.getConfig()
         then:
         config.getSecret() == []
@@ -193,7 +193,7 @@ class ProcessDslTest extends Specification {
 
     def 'should set process labels'() {
         when:
-        def builder = new ProcessDsl(Mock(BaseScript), null)
+        def builder = new ProcessBuilder(Mock(BaseScript), null)
         def config = builder.getConfig()
         then:
         config.getLabels() == []
@@ -211,7 +211,7 @@ class ProcessDslTest extends Specification {
 
     def 'should apply resource labels config' () {
         given:
-        def builder = new ProcessDsl(Mock(BaseScript), null)
+        def builder = new ProcessBuilder(Mock(BaseScript), null)
         def config = builder.getConfig()
         expect:
         config.getResourceLabels() == [:]
@@ -231,7 +231,7 @@ class ProcessDslTest extends Specification {
     def 'should check a valid label' () {
 
         expect:
-        ProcessDsl.isValidLabel(lbl) == result
+        ProcessBuilder.isValidLabel(lbl) == result
 
         where:
         lbl         | result
@@ -261,7 +261,7 @@ class ProcessDslTest extends Specification {
     @Unroll
     def 'should match selector: #SELECTOR with #TARGET' () {
         expect:
-        ProcessDsl.matchesSelector(TARGET, SELECTOR) == EXPECTED
+        ProcessBuilder.matchesSelector(TARGET, SELECTOR) == EXPECTED
 
         where:
         SELECTOR        | TARGET    | EXPECTED
@@ -288,7 +288,7 @@ class ProcessDslTest extends Specification {
 
         when:
         def config = new ProcessConfig([:])
-        new ProcessDsl(config).applyConfigSelectorWithLabels(settings, ['short'])
+        new ProcessBuilder(config).applyConfigSelectorWithLabels(settings, ['short'])
         then:
         config.cpus == 1
         config.time == '1h'
@@ -296,7 +296,7 @@ class ProcessDslTest extends Specification {
 
         when:
         config = new ProcessConfig([:])
-        new ProcessDsl(config).applyConfigSelectorWithLabels(settings, ['long'])
+        new ProcessBuilder(config).applyConfigSelectorWithLabels(settings, ['long'])
         then:
         config.cpus == 32
         config.queue == 'cn-long'
@@ -304,7 +304,7 @@ class ProcessDslTest extends Specification {
 
         when:
         config = new ProcessConfig([:])
-        new ProcessDsl(config).applyConfigSelectorWithLabels(settings, ['foo'])
+        new ProcessBuilder(config).applyConfigSelectorWithLabels(settings, ['foo'])
         then:
         config.cpus == 2
         config.disk == '100GB'
@@ -313,7 +313,7 @@ class ProcessDslTest extends Specification {
 
         when:
         config = new ProcessConfig([:])
-        new ProcessDsl(config).applyConfigSelectorWithLabels(settings, ['bar'])
+        new ProcessBuilder(config).applyConfigSelectorWithLabels(settings, ['bar'])
         then:
         config.cpus == 32
         config.disk == '100GB'
@@ -322,7 +322,7 @@ class ProcessDslTest extends Specification {
 
         when:
         config = new ProcessConfig([:])
-        new ProcessDsl(config).applyConfigSelectorWithLabels(settings, ['gpu-1'])
+        new ProcessBuilder(config).applyConfigSelectorWithLabels(settings, ['gpu-1'])
         then:
         config.cpus == 4
         config.queue == 'cn-long'
@@ -342,13 +342,13 @@ class ProcessDslTest extends Specification {
 
         when:
         def config = new ProcessConfig([:])
-        new ProcessDsl(config).applyConfigSelectorWithName(settings, 'xx')
+        new ProcessBuilder(config).applyConfigSelectorWithName(settings, 'xx')
         then:
         config.size() == 0
 
         when:
         config = new ProcessConfig([:])
-        new ProcessDsl(config).applyConfigSelectorWithName(settings, 'alpha')
+        new ProcessBuilder(config).applyConfigSelectorWithName(settings, 'alpha')
         then:
         config.cpus == 1
         config.time == '1h'
@@ -356,7 +356,7 @@ class ProcessDslTest extends Specification {
 
         when:
         config = new ProcessConfig([:])
-        new ProcessDsl(config).applyConfigSelectorWithName(settings, 'delta')
+        new ProcessBuilder(config).applyConfigSelectorWithName(settings, 'delta')
         then:
         config.cpus == 2
         config.disk == '100GB'
@@ -364,14 +364,14 @@ class ProcessDslTest extends Specification {
 
         when:
         config = new ProcessConfig([:])
-        new ProcessDsl(config).applyConfigSelectorWithName(settings, 'gamma')
+        new ProcessBuilder(config).applyConfigSelectorWithName(settings, 'gamma')
         then:
         config.disk == '100GB'
         config.size() == 1
 
         when:
         config = new ProcessConfig([:])
-        new ProcessDsl(config).applyConfigSelectorWithName(settings, 'omega_x')
+        new ProcessBuilder(config).applyConfigSelectorWithName(settings, 'omega_x')
         then:
         config.cpus == 4
         config.size() == 1
@@ -381,7 +381,7 @@ class ProcessDslTest extends Specification {
     def 'should apply config process defaults' () {
 
         when:
-        def builder = new ProcessDsl(Mock(BaseScript), null)
+        def builder = new ProcessBuilder(Mock(BaseScript), null)
         builder.queue 'cn-el6'
         builder.memory '10 GB'
         builder.applyConfigDefaults(
@@ -399,7 +399,7 @@ class ProcessDslTest extends Specification {
 
 
         when:
-        builder = new ProcessDsl(Mock(BaseScript), null)
+        builder = new ProcessBuilder(Mock(BaseScript), null)
         builder.container null
         builder.applyConfigDefaults(
                 queue: 'def-queue',
@@ -415,7 +415,7 @@ class ProcessDslTest extends Specification {
 
 
         when:
-        builder = new ProcessDsl(Mock(BaseScript), null)
+        builder = new ProcessBuilder(Mock(BaseScript), null)
         builder.maxRetries 10
         builder.applyConfigDefaults(
                 queue: 'def-queue',
@@ -433,7 +433,7 @@ class ProcessDslTest extends Specification {
     def 'should apply pod configs' () {
 
         when:
-        def builder = new ProcessDsl(Mock(BaseScript), null)
+        def builder = new ProcessBuilder(Mock(BaseScript), null)
         builder.applyConfigDefaults( pod: [secret: 'foo', mountPath: '/there'] )
         then:
         builder.getConfig().pod == [
@@ -441,7 +441,7 @@ class ProcessDslTest extends Specification {
         ]
 
         when:
-        builder = new ProcessDsl(Mock(BaseScript), null)
+        builder = new ProcessBuilder(Mock(BaseScript), null)
         builder.applyConfigDefaults( pod: [
                 [secret: 'foo', mountPath: '/here'],
                 [secret: 'bar', mountPath: '/there']
@@ -457,7 +457,7 @@ class ProcessDslTest extends Specification {
     def 'should clone config object' () {
 
         when:
-        def builder = new ProcessDsl(Mock(BaseScript), null)
+        def builder = new ProcessBuilder(Mock(BaseScript), null)
         def config = builder.getConfig()
         builder.queue 'cn-el6'
         builder.container 'ubuntu:latest'
@@ -475,7 +475,7 @@ class ProcessDslTest extends Specification {
 
         when:
         def copy = config.clone()
-        builder = new ProcessDsl(copy)
+        builder = new ProcessBuilder(copy)
         builder.queue 'long'
         builder.container 'debian:wheezy'
         builder.memory '5 GB'
@@ -500,7 +500,7 @@ class ProcessDslTest extends Specification {
     def 'should apply accelerator config' () {
 
         given:
-        def builder = new ProcessDsl(Mock(BaseScript), null)
+        def builder = new ProcessBuilder(Mock(BaseScript), null)
         def config = builder.getConfig()
 
         when:
@@ -532,7 +532,7 @@ class ProcessDslTest extends Specification {
     def 'should apply disk config' () {
 
         given:
-        def builder = new ProcessDsl(Mock(BaseScript), null)
+        def builder = new ProcessBuilder(Mock(BaseScript), null)
         def config = builder.getConfig()
 
         when:
@@ -554,7 +554,7 @@ class ProcessDslTest extends Specification {
     def 'should apply architecture config' () {
 
         given:
-        def builder = new ProcessDsl(Mock(BaseScript), null)
+        def builder = new ProcessBuilder(Mock(BaseScript), null)
         def config = builder.getConfig()
 
         when:
@@ -583,21 +583,21 @@ class ProcessDslTest extends Specification {
 
         when:
         def config = new ProcessConfig(label: ['foo', 'other'])
-        new ProcessDsl(config).applyConfig(settings, "processName", null, null)
+        new ProcessBuilder(config).applyConfig(settings, "processName", null, null)
         then:
         config.cpus == 2
         config.disk == '100.GB'
 
         when:
         config = new ProcessConfig(label: ['foo', 'other', 'nodisk_label'])
-        new ProcessDsl(config).applyConfig(settings, "processName", null, null)
+        new ProcessBuilder(config).applyConfig(settings, "processName", null, null)
         then:
         config.cpus == 2
         !config.disk
 
         when:
         config = new ProcessConfig(label: ['other', 'nodisk_label'])
-        new ProcessDsl(config).applyConfig(settings, "processName", null, null)
+        new ProcessBuilder(config).applyConfig(settings, "processName", null, null)
         then:
         config.cpus == 4
         !config.disk
@@ -606,7 +606,7 @@ class ProcessDslTest extends Specification {
 
     def 'should throw exception for invalid error strategy' () {
         when:
-        def builder = new ProcessDsl(Mock(BaseScript), null)
+        def builder = new ProcessBuilder(Mock(BaseScript), null)
         builder.errorStrategy 'abort'
 
         then:
@@ -617,21 +617,21 @@ class ProcessDslTest extends Specification {
 
     def 'should not throw exception for valid error strategy or closure' () {
         when:
-        def builder = new ProcessDsl(Mock(BaseScript), null)
+        def builder = new ProcessBuilder(Mock(BaseScript), null)
         builder.errorStrategy 'retry'
 
         then:
         noExceptionThrown()
 
         when:
-        builder = new ProcessDsl(Mock(BaseScript), null)
+        builder = new ProcessBuilder(Mock(BaseScript), null)
         builder.errorStrategy 'terminate'
 
         then:
         noExceptionThrown()
 
         when:
-        builder = new ProcessDsl(Mock(BaseScript), null)
+        builder = new ProcessBuilder(Mock(BaseScript), null)
         builder.errorStrategy { task.exitStatus==14 ? 'retry' : 'terminate' }
 
         then:

From c4650004799939ff4a95fd5cc69fd8f1f5cd4b50 Mon Sep 17 00:00:00 2001
From: Ben Sherman 
Date: Thu, 30 Nov 2023 08:13:42 -0600
Subject: [PATCH 05/36] Add WorkflowFn annotation

Signed-off-by: Ben Sherman 
---
 .../nextflow/ast/NextflowXformImpl.groovy     |   1 +
 .../groovy/nextflow/ast/ProcessFnXform.groovy |  15 +-
 .../groovy/nextflow/ast/WorkflowFn.groovy     |  37 ++++
 .../nextflow/ast/WorkflowFnXform.groovy       |  90 +++++++++
 .../nextflow/processor/TaskProcessor.groovy   |   2 +-
 .../groovy/nextflow/script/BaseScript.groovy  | 180 +++++++++++-------
 .../groovy/nextflow/script/ScriptMeta.groovy  |   2 +
 .../nextflow/script/ScriptParser.groovy       |   2 +
 .../groovy/nextflow/script/WorkflowDef.groovy |  27 ++-
 ...kflowDsl.groovy => WorkflowBuilder.groovy} |  13 +-
 10 files changed, 292 insertions(+), 77 deletions(-)
 create mode 100644 modules/nextflow/src/main/groovy/nextflow/ast/WorkflowFn.groovy
 create mode 100644 modules/nextflow/src/main/groovy/nextflow/ast/WorkflowFnXform.groovy
 rename modules/nextflow/src/main/groovy/nextflow/script/dsl/{WorkflowDsl.groovy => WorkflowBuilder.groovy} (86%)

diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/NextflowXformImpl.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/NextflowXformImpl.groovy
index f5dfe8abed..4d3c4fa2a5 100644
--- a/modules/nextflow/src/main/groovy/nextflow/ast/NextflowXformImpl.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/ast/NextflowXformImpl.groovy
@@ -48,6 +48,7 @@ class NextflowXformImpl implements ASTTransformation {
         new DslCodeVisitor(unit).visitClass(classNode)
         new OperatorXform(unit).visitClass(classNode)
         new ProcessFnXform(unit).visitClass(classNode)
+        new WorkflowFnXform(unit).visitClass(classNode)
     }
 
 }
diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/ProcessFnXform.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/ProcessFnXform.groovy
index 938c229536..6a04d1af07 100644
--- a/modules/nextflow/src/main/groovy/nextflow/ast/ProcessFnXform.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/ast/ProcessFnXform.groovy
@@ -22,6 +22,7 @@ import groovy.transform.CompileStatic
 import groovy.util.logging.Slf4j
 import nextflow.processor.TaskConfig
 import org.codehaus.groovy.ast.ASTNode
+import org.codehaus.groovy.ast.AnnotationNode
 import org.codehaus.groovy.ast.ClassCodeVisitorSupport
 import org.codehaus.groovy.ast.ClassNode
 import org.codehaus.groovy.ast.MethodNode
@@ -64,18 +65,18 @@ class ProcessFnXform extends ClassCodeVisitorSupport {
                 .find(a -> a.getClassNode().getName() == 'ProcessFn')
 
         if( annotation )
-            transform(method, annotation.getMembers())
+            transform(method, annotation)
     }
 
     /**
      * Transform a ProcessFn annotation to be semantically valid.
      *
      * @param method
-     * @param opts
+     * @param annotation
      */
-    protected void transform(MethodNode method, Map opts) {
+    protected void transform(MethodNode method, AnnotationNode annotation) {
         // fix directives
-        final directives = opts['directives']
+        final directives = annotation.getMember('directives')
         if( directives != null && directives instanceof ClosureExpression ) {
             final block = (BlockStatement)directives.getCode()
             for( Statement stmt : block.getStatements() )
@@ -83,7 +84,7 @@ class ProcessFnXform extends ClassCodeVisitorSupport {
         }
 
         // fix outputs
-        final outputs = opts['outputs']
+        final outputs = annotation.getMember('outputs')
         if( outputs != null && outputs instanceof ClosureExpression ) {
             final block = (BlockStatement)outputs.getCode()
             for( Statement stmt : block.getStatements() )
@@ -98,14 +99,14 @@ class ProcessFnXform extends ClassCodeVisitorSupport {
         // TODO: append stub source
 
         // append method params
-        opts.put( 'params', closureX( block( new ExpressionStatement(
+        annotation.addMember( 'params', closureX( block( new ExpressionStatement(
             new ListExpression(
                 params.collect(p -> (Expression)constX(p.getName()))
             )
         ) ) ) )
 
         // append script source
-        opts.put( 'source', constX( getSource(method.getCode()) ) )
+        annotation.addMember( 'source', constX( getSource(method.getCode()) ) )
     }
 
     /**
diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/WorkflowFn.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/WorkflowFn.groovy
new file mode 100644
index 0000000000..ca9ef36f4c
--- /dev/null
+++ b/modules/nextflow/src/main/groovy/nextflow/ast/WorkflowFn.groovy
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2013-2023, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package nextflow.ast
+
+import java.lang.annotation.ElementType
+import java.lang.annotation.Retention
+import java.lang.annotation.RetentionPolicy
+import java.lang.annotation.Target
+
+/**
+ * Annotation for workflow functions.
+ *
+ * @author Ben Sherman 
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.METHOD)
+@interface WorkflowFn {
+    boolean main() default false
+
+    // injected via AST transform
+    Class params()
+    String source()
+}
diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/WorkflowFnXform.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/WorkflowFnXform.groovy
new file mode 100644
index 0000000000..f57d339dd2
--- /dev/null
+++ b/modules/nextflow/src/main/groovy/nextflow/ast/WorkflowFnXform.groovy
@@ -0,0 +1,90 @@
+/*
+ * Copyright 2013-2023, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package nextflow.ast
+
+import static org.codehaus.groovy.ast.tools.GeneralUtils.*
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.codehaus.groovy.ast.ASTNode
+import org.codehaus.groovy.ast.AnnotationNode
+import org.codehaus.groovy.ast.ClassCodeVisitorSupport
+import org.codehaus.groovy.ast.MethodNode
+import org.codehaus.groovy.ast.Parameter
+import org.codehaus.groovy.ast.expr.Expression
+import org.codehaus.groovy.ast.expr.ListExpression
+import org.codehaus.groovy.ast.stmt.ExpressionStatement
+import org.codehaus.groovy.control.SourceUnit
+/**
+ * Implements syntax transformations for workflow functions.
+ *
+ * @author Ben Sherman 
+ */
+@Slf4j
+@CompileStatic
+class WorkflowFnXform extends ClassCodeVisitorSupport {
+
+    private final SourceUnit unit
+
+    WorkflowFnXform(SourceUnit unit) {
+        this.unit = unit
+    }
+
+    @Override
+    protected SourceUnit getSourceUnit() { unit }
+
+    @Override
+    void visitMethod(MethodNode method) {
+        final annotation = method.getAnnotations()
+                .find(a -> a.getClassNode().getName() == 'WorkflowFn')
+
+        if( annotation )
+            transform(method, annotation)
+    }
+
+    protected void transform(MethodNode method, AnnotationNode annotation) {
+        // append method params
+        final params = method.getParameters() as List
+        annotation.addMember( 'params', closureX( block( new ExpressionStatement(
+            new ListExpression(
+                params.collect(p -> (Expression)constX(p.getName()))
+            )
+        ) ) ) )
+
+        // append workflow source
+        annotation.addMember( 'source', constX( getSource(method.getCode()) ) )
+    }
+
+    private String getSource(ASTNode node) {
+        final buffer = new StringBuilder()
+        final colx = node.getColumnNumber()
+        final colz = node.getLastColumnNumber()
+        final first = node.getLineNumber()
+        final last = node.getLastLineNumber()
+        for( int i=first; i<=last; i++ ) {
+            def line = unit.source.getLine(i, null)
+            if( i==last )
+                line = line.substring(0,colz-1)
+            if( i==first )
+                line = line.substring(colx-1)
+            buffer.append(line) .append('\n')
+        }
+
+        return buffer.toString()
+    }
+
+}
diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy
index 495ecf0f63..bac9ed688c 100644
--- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy
@@ -2227,7 +2227,7 @@ class TaskProcessor {
     }
 
     protected Map getTaskGlobalVars(TaskRun task) {
-        final result = task.getGlobalVars(ownerScript.binding)
+        final result = task.getGlobalVars(ownerScript.getBinding())
         final directives = getTaskExtensionDirectiveVars(task)
         result.putAll(directives)
         return result
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy b/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy
index 50a1f8b219..d684791dbd 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy
@@ -17,6 +17,7 @@
 package nextflow.script
 
 import java.lang.reflect.InvocationTargetException
+import java.lang.reflect.Method
 import java.nio.file.Paths
 
 import groovy.transform.CompileStatic
@@ -24,16 +25,18 @@ import groovy.util.logging.Slf4j
 import nextflow.NextflowMeta
 import nextflow.Session
 import nextflow.ast.ProcessFn
+import nextflow.ast.WorkflowFn
 import nextflow.exception.AbortOperationException
 import nextflow.script.dsl.ProcessBuilder
 import nextflow.script.dsl.ProcessInputsBuilder
-import nextflow.script.dsl.WorkflowDsl
+import nextflow.script.dsl.WorkflowBuilder
 /**
  * Any user defined script will extends this class, it provides the base execution context
  *
  * @author Paolo Di Tommaso 
  */
 @Slf4j
+@CompileStatic
 abstract class BaseScript extends Script implements ExecutionContext {
 
     private Session session
@@ -115,12 +118,7 @@ abstract class BaseScript extends Script implements ExecutionContext {
      * @param rawBody
      */
     protected void workflow(Closure rawBody) {
-        final builder = new WorkflowDsl(this)
-        final copy = (Closure)rawBody.clone()
-        copy.delegate = builder
-        copy.resolveStrategy = Closure.DELEGATE_FIRST
-        final body = copy.call()
-        final workflow = builder.withBody(body).build()
+        final workflow = workflow0(null, rawBody)
         this.entryFlow = workflow
         meta.addDefinition(workflow)
     }
@@ -132,13 +130,17 @@ abstract class BaseScript extends Script implements ExecutionContext {
      * @param rawBody
      */
     protected void workflow(String name, Closure rawBody) {
-        final builder = new WorkflowDsl(this, name)
+        final workflow = workflow0(name, rawBody)
+        meta.addDefinition(workflow)
+    }
+
+    protected WorkflowDef workflow0(String name, Closure rawBody) {
+        final builder = new WorkflowBuilder(this, name)
         final copy = (Closure)rawBody.clone()
         copy.delegate = builder
         copy.resolveStrategy = Closure.DELEGATE_FIRST
         final body = copy.call()
-        final workflow = builder.withBody(body).build()
-        meta.addDefinition(workflow)
+        return builder.withBody(body).build()
     }
 
     protected IncludeDef include( IncludeDef include ) {
@@ -147,63 +149,113 @@ abstract class BaseScript extends Script implements ExecutionContext {
         include .setSession(session)
     }
 
-    @CompileStatic
-    private void registerProcessFunctions() {
-        final dslEval = { Object delegate, Class clazz ->
-            final cl = clazz.newInstance(this, this)
-            cl.delegate = delegate
-            cl.resolveStrategy = Closure.DELEGATE_FIRST
-            cl.call()
+    @Override
+    Object getProperty(String name) {
+        try {
+            ExecutionStack.binding().getProperty(name)
         }
-
-        final clazz = this.getClass()
-        for( final method : clazz.getDeclaredMethods() ) {
-            // check for ProcessFn annotation
-            final name = method.getName()
-            final processFn = method.getAnnotation(ProcessFn)
-            if( !processFn )
-                continue
-
-            // validate annotation
-            if( processFn.script() && processFn.shell() )
-                throw new IllegalArgumentException("Process function `${name}` cannot have script and shell enabled simultaneously")
-
-            // build process from annotation
-            final builder = new ProcessBuilder(this, name)
-            final inputsBuilder = new ProcessInputsBuilder(builder.getConfig())
-
-            dslEval(inputsBuilder, processFn.inputs())
-            dslEval(builder, processFn.directives())
-            dslEval(builder, processFn.outputs())
-
-            // get method parameters
-            final paramNames = (List)((Closure)processFn.params().newInstance(this, this)).call()
-            final params = (0 ..< paramNames.size()).collect( i ->
-                new Parameter( paramNames[i], method.getParameters()[i].getType() )
-            )
-
-            builder.config.params = params
-
-            // determine process type
-            def type
-            if( processFn.script() )
-                type = 'script'
-            else if( processFn.shell() )
-                type = 'shell'
-            else
-                type = 'exec'
-
-            // create task body
-            final taskBody = new BodyDef( this.&"${name}", processFn.source(), type, [] )
-
-            final process = builder.withBody(taskBody).build()
-            meta.addDefinition(process)
+        catch( MissingPropertyException e ) {
+            if( !ExecutionStack.withinWorkflow() )
+                throw e
+            binding.getProperty(name)
         }
     }
 
+    /**
+     * Invokes custom methods in the task execution context
+     *
+     * @see nextflow.processor.TaskContext#invokeMethod(java.lang.String, java.lang.Object)
+     * @see WorkflowBinding#invokeMethod(java.lang.String, java.lang.Object)
+     *
+     * @param name the name of the method to call
+     * @param args the arguments to use for the method call
+     * @return The result of the custom method execution
+     */
+    @Override
+    Object invokeMethod(String name, Object args) {
+        ExecutionStack.binding().invokeMethod(name, args)
+    }
+
+    private void applyDsl(Object delegate, Class clazz) {
+        final cl = clazz.newInstance(this, this)
+        cl.delegate = delegate
+        cl.resolveStrategy = Closure.DELEGATE_FIRST
+        cl.call()
+    }
+
+    private void registerProcessFn(Method method) {
+        final name = method.getName()
+        final processFn = method.getAnnotation(ProcessFn)
+
+        // validate annotation
+        if( processFn.script() && processFn.shell() )
+            throw new IllegalArgumentException("Process function `${name}` cannot have script and shell enabled simultaneously")
+
+        // build process from annotation
+        final builder = new ProcessBuilder(this, name)
+        final inputsBuilder = new ProcessInputsBuilder(builder.getConfig())
+
+        applyDsl(inputsBuilder, processFn.inputs())
+        applyDsl(builder, processFn.directives())
+        applyDsl(builder, processFn.outputs())
+
+        // get method parameters
+        final paramNames = (List)((Closure)processFn.params().newInstance(this, this)).call()
+        final params = (0 ..< paramNames.size()).collect( i ->
+            new Parameter( paramNames[i], method.getParameters()[i].getType() )
+        )
+        builder.config.params = params
+
+        // determine process type
+        def type
+        if( processFn.script() )
+            type = 'script'
+        else if( processFn.shell() )
+            type = 'shell'
+        else
+            type = 'exec'
+
+        // create task body
+        final taskBody = new BodyDef( this.&"${name}", processFn.source(), type, [] )
+        builder.withBody(taskBody)
+
+        // register process
+        meta.addDefinition(builder.build())
+    }
+
+    private void registerWorkflowFn(Method method) {
+        final name = method.getName()
+        final workflowFn = method.getAnnotation(WorkflowFn)
+
+        // build workflow from annotation
+        final builder = workflowFn.main()
+            ? new WorkflowBuilder(this)
+            : new WorkflowBuilder(this, name)
+
+        // get method parameters
+        final params = (List)((Closure)workflowFn.params().newInstance(this, this)).call()
+        builder.withParams(params)
+
+        // create body
+        final body = new BodyDef( this.&"${name}", workflowFn.source(), 'workflow', [] )
+        builder.withBody(body)
+
+        // register workflow
+        final workflow = builder.build()
+        if( workflowFn.main() )
+            this.entryFlow = workflow
+        meta.addDefinition(workflow)
+    }
+
     private run0() {
-        // register any process functions
-        registerProcessFunctions()
+        // register any process and workflow functions
+        final clazz = this.getClass()
+        for( final method : clazz.getDeclaredMethods() ) {
+            if( method.isAnnotationPresent(ProcessFn) )
+                registerProcessFn(method)
+            if( method.isAnnotationPresent(WorkflowFn) )
+                registerWorkflowFn(method)
+        }
 
         // execute script
         final result = runScript()
@@ -309,7 +361,7 @@ abstract class BaseScript extends Script implements ExecutionContext {
             return
 
         if( session?.ansiLog )
-            log.info(String.printf(msg, arg))
+            log.info(String.format(msg, arg))
         else
             super.printf(msg, arg)
     }
@@ -320,7 +372,7 @@ abstract class BaseScript extends Script implements ExecutionContext {
             return
 
         if( session?.ansiLog )
-            log.info(String.printf(msg, args))
+            log.info(String.format(msg, args))
         else
             super.printf(msg, args)
     }
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ScriptMeta.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ScriptMeta.groovy
index 93a40ee2e6..3b9e5dd622 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ScriptMeta.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ScriptMeta.groovy
@@ -26,6 +26,7 @@ import groovy.transform.PackageScope
 import groovy.util.logging.Slf4j
 import nextflow.NF
 import nextflow.ast.ProcessFn
+import nextflow.ast.WorkflowFn
 import nextflow.exception.DuplicateModuleFunctionException
 import nextflow.exception.MissingModuleComponentException
 import nextflow.script.bundle.ResourcesBundle
@@ -177,6 +178,7 @@ class ScriptMeta {
             if( method.name.startsWith('super$')) continue
             if( method.name in INVALID_FUNCTION_NAMES ) continue
             if( method.isAnnotationPresent(ProcessFn) ) continue
+            if( method.isAnnotationPresent(WorkflowFn) ) continue
 
             // If method is already into the list, maybe with other signature, it's not necessary to include it again
             if( result.find{it.name == method.name}) continue
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ScriptParser.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ScriptParser.groovy
index a3801fa65a..05eb862b3b 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ScriptParser.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ScriptParser.groovy
@@ -25,6 +25,7 @@ import nextflow.Nextflow
 import nextflow.Session
 import nextflow.ast.NextflowXform
 import nextflow.ast.ProcessFn
+import nextflow.ast.WorkflowFn
 import nextflow.exception.ScriptCompilationException
 import nextflow.extension.FilesEx
 import nextflow.file.FileHelper
@@ -115,6 +116,7 @@ class ScriptParser {
         importCustomizer.addImports( MemoryUnit.name )
         importCustomizer.addImports( ProcessFn.name )
         importCustomizer.addImports( ValueObject.name )
+        importCustomizer.addImports( WorkflowFn.name )
         importCustomizer.addImport( 'channel', Channel.name )
         importCustomizer.addStaticStars( Nextflow.name )
 
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/WorkflowDef.groovy b/modules/nextflow/src/main/groovy/nextflow/script/WorkflowDef.groovy
index 3b5d4b5cf8..bdeb901089 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/WorkflowDef.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/WorkflowDef.groovy
@@ -16,6 +16,7 @@
 
 package nextflow.script
 
+import groovy.transform.CompileDynamic
 import groovy.transform.CompileStatic
 import groovy.transform.PackageScope
 import groovy.util.logging.Slf4j
@@ -189,16 +190,40 @@ class WorkflowDef extends BindableDef implements ChainableDef, IterableDef, Exec
         }
     }
 
+    @CompileDynamic
     private Object run0(Object[] args) {
         collectInputs(binding, args)
         // invoke the workflow execution
         final closure = body.closure
         closure.delegate = binding
         closure.setResolveStrategy(Closure.DELEGATE_FIRST)
-        closure.call()
+        def result = closure.call(*args)
+
+        // apply return value to declared outputs, binding
+        normalizeOutput(result)
+
         // collect the workflow outputs
         output = collectOutputs(declaredOutputs)
         return output
     }
 
+    private void normalizeOutput(Object result) {
+        if( CH.isChannel(result) )
+            result = ['$out0': result]
+
+        if( result instanceof List )
+            result = result.inject([:], (acc, value) -> { acc.put("\$out${acc.size()}".toString(), value); acc })
+
+        if( result instanceof Map ) {
+            for( def entry : result ) {
+                declaredOutputs.add(entry.key)
+                binding.setVariable(entry.key, entry.value)
+            }
+        }
+        else if( result instanceof ChannelOut )
+            log.debug "Workflow `$name` > ignoring multi-channel return value"
+        else if( result != null )
+            throw new ScriptRuntimeException("Workflow `$name` emitted unexpected value of type ${result.class.name} -- ${result}")
+    }
+
 }
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/dsl/WorkflowDsl.groovy b/modules/nextflow/src/main/groovy/nextflow/script/dsl/WorkflowBuilder.groovy
similarity index 86%
rename from modules/nextflow/src/main/groovy/nextflow/script/dsl/WorkflowDsl.groovy
rename to modules/nextflow/src/main/groovy/nextflow/script/dsl/WorkflowBuilder.groovy
index 1706201c6e..506a2895e3 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/dsl/WorkflowDsl.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/dsl/WorkflowBuilder.groovy
@@ -22,13 +22,13 @@ import nextflow.script.BaseScript
 import nextflow.script.BodyDef
 import nextflow.script.WorkflowDef
 /**
- * Implements the workflow DSL.
+ * Implements the workflow builder DSL.
  *
  * @author Ben Sherman 
  */
 @Slf4j
 @CompileStatic
-class WorkflowDsl {
+class WorkflowBuilder {
 
     static final private String TAKE_PREFIX = '_take_'
     static final private String EMIT_PREFIX = '_emit_'
@@ -39,7 +39,7 @@ class WorkflowDsl {
     private Map takes = new LinkedHashMap<>(10)
     private Map emits = new LinkedHashMap<>(10)
 
-    WorkflowDsl(BaseScript owner, String name=null) {
+    WorkflowBuilder(BaseScript owner, String name=null) {
         this.owner = owner
         this.name = name
     }
@@ -56,7 +56,12 @@ class WorkflowDsl {
             throw new MissingMethodException(name, WorkflowDef, args)
     }
 
-    WorkflowDsl withBody(BodyDef body) {
+    WorkflowBuilder withParams(List params) {
+        for( String param : params )
+            takes.put(param, true)
+    }
+
+    WorkflowBuilder withBody(BodyDef body) {
         this.body = body
         return this
     }

From 8f2c0902833877c589d652a36ee846c898f97f87 Mon Sep 17 00:00:00 2001
From: Ben Sherman 
Date: Thu, 30 Nov 2023 19:16:35 -0600
Subject: [PATCH 06/36] Add support for native processes, use reflection to
 invoke workflows

Signed-off-by: Ben Sherman 
---
 .../main/groovy/nextflow/ast/ProcessFn.groovy |  6 ++---
 .../groovy/nextflow/processor/TaskRun.groovy  | 12 +++++----
 .../groovy/nextflow/script/BaseScript.groovy  |  9 ++++++-
 .../groovy/nextflow/script/WorkflowDef.groovy | 27 ++++++++++++++-----
 4 files changed, 39 insertions(+), 15 deletions(-)

diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/ProcessFn.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/ProcessFn.groovy
index c00e45c8fc..203118b061 100644
--- a/modules/nextflow/src/main/groovy/nextflow/ast/ProcessFn.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/ast/ProcessFn.groovy
@@ -29,9 +29,9 @@ import java.lang.annotation.Target
 @Retention(RetentionPolicy.RUNTIME)
 @Target(ElementType.METHOD)
 @interface ProcessFn {
-    Class directives()
-    Class inputs()
-    Class outputs()
+    Class directives() default {->}
+    Class inputs() default {->}
+    Class outputs() default {->}
 
     boolean script() default false
     boolean shell() default false
diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy
index ec2842cecf..54e00dbf4e 100644
--- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy
@@ -725,17 +725,19 @@ class TaskRun implements Cloneable {
         // note: this may be overwritten when a template file is used
         this.source = body.source
 
-        if( body.type != ScriptType.SCRIPTLET )
-            return
+        // -- collect method args from task context
+        final args = params.collect(param -> context[param]).toArray()
 
-        // collect method args from task context
-        final args = params.collect(param -> context[param])
+        if( body.type != ScriptType.SCRIPTLET ) {
+            code = code.curry(args)
+            return
+        }
 
         // Important!
         // when the task is implemented by a script string
         // Invoke the closure which returns the script with all the variables replaced with the actual values
         try {
-            final result = code.call(*args)
+            final result = code.call(args)
             if ( result instanceof Path ) {
                 script = renderTemplate(result, body.isShell)
             }
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy b/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy
index d684791dbd..64170181f2 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy
@@ -173,7 +173,14 @@ abstract class BaseScript extends Script implements ExecutionContext {
      */
     @Override
     Object invokeMethod(String name, Object args) {
-        ExecutionStack.binding().invokeMethod(name, args)
+        try {
+            ExecutionStack.binding().invokeMethod(name, args)
+        }
+        catch( MissingMethodException e ) {
+            if( !ExecutionStack.withinWorkflow() )
+                throw e
+            binding.invokeMethod(name, args)
+        }
     }
 
     private void applyDsl(Object delegate, Class clazz) {
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/WorkflowDef.groovy b/modules/nextflow/src/main/groovy/nextflow/script/WorkflowDef.groovy
index bdeb901089..7a294a17b3 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/WorkflowDef.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/WorkflowDef.groovy
@@ -16,7 +16,6 @@
 
 package nextflow.script
 
-import groovy.transform.CompileDynamic
 import groovy.transform.CompileStatic
 import groovy.transform.PackageScope
 import groovy.util.logging.Slf4j
@@ -25,6 +24,7 @@ import nextflow.exception.MissingProcessException
 import nextflow.exception.MissingValueException
 import nextflow.exception.ScriptRuntimeException
 import nextflow.extension.CH
+import org.codehaus.groovy.runtime.MethodClosure
 /**
  * Models a script workflow component
  *
@@ -190,14 +190,10 @@ class WorkflowDef extends BindableDef implements ChainableDef, IterableDef, Exec
         }
     }
 
-    @CompileDynamic
     private Object run0(Object[] args) {
         collectInputs(binding, args)
         // invoke the workflow execution
-        final closure = body.closure
-        closure.delegate = binding
-        closure.setResolveStrategy(Closure.DELEGATE_FIRST)
-        def result = closure.call(*args)
+        final result = run1(body.closure, args)
 
         // apply return value to declared outputs, binding
         normalizeOutput(result)
@@ -207,6 +203,25 @@ class WorkflowDef extends BindableDef implements ChainableDef, IterableDef, Exec
         return output
     }
 
+    private Object run1(Closure closure, Object[] args) {
+        if( closure instanceof MethodClosure ) {
+            final target = closure.owner
+            final name = closure.method
+            final meta = target.metaClass.getMetaMethod(name, args)
+            if( meta == null )
+                throw new MissingMethodException(name, target.getClass(), args)
+            final method = target.getClass().getMethod(name, meta.getNativeParameterTypes())
+            if( method == null )
+                throw new MissingMethodException(name, target.getClass(), args)
+            return method.invoke(target, args)
+        }
+        else {
+            closure.setDelegate(binding)
+            closure.setResolveStrategy(Closure.DELEGATE_FIRST)
+            return closure.call()
+        }
+    }
+
     private void normalizeOutput(Object result) {
         if( CH.isChannel(result) )
             result = ['$out0': result]

From 48fdfc23fea4f0417b5409d309af110e380d969f Mon Sep 17 00:00:00 2001
From: Ben Sherman 
Date: Fri, 1 Dec 2023 16:25:24 -0600
Subject: [PATCH 07/36] Separate process input channel logic from task
 processor

Signed-off-by: Ben Sherman 
---
 .../src/main/groovy/nextflow/dag/DAG.groovy   |   5 +-
 .../nextflow/processor/ForwardClosure.groovy  | 107 ---------
 .../nextflow/processor/TaskProcessor.groovy   | 219 +++---------------
 .../nextflow/script/ProcessConfig.groovy      |   9 -
 .../groovy/nextflow/script/ProcessDef.groovy  |  54 +++--
 .../nextflow/script/dsl/ProcessBuilder.groovy |   6 +
 .../nextflow/script/params/BaseInParam.groovy |   5 -
 .../script/params/DefaultInParam.groovy       |  42 ----
 8 files changed, 83 insertions(+), 364 deletions(-)
 delete mode 100644 modules/nextflow/src/main/groovy/nextflow/processor/ForwardClosure.groovy
 delete mode 100644 modules/nextflow/src/main/groovy/nextflow/script/params/DefaultInParam.groovy

diff --git a/modules/nextflow/src/main/groovy/nextflow/dag/DAG.groovy b/modules/nextflow/src/main/groovy/nextflow/dag/DAG.groovy
index e61dac5985..bc92a3b3ce 100644
--- a/modules/nextflow/src/main/groovy/nextflow/dag/DAG.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/dag/DAG.groovy
@@ -30,7 +30,6 @@ import nextflow.NF
 import nextflow.extension.CH
 import nextflow.extension.DataflowHelper
 import nextflow.processor.TaskProcessor
-import nextflow.script.params.DefaultInParam
 import nextflow.script.params.DefaultOutParam
 import nextflow.script.params.EachInParam
 import nextflow.script.params.InParam
@@ -237,9 +236,7 @@ class DAG {
 
     private List normalizeInputs( InputsList inputs ) {
 
-        inputs
-                .findAll { !( it instanceof DefaultInParam)  }
-                .collect { InParam p -> new ChannelHandler(channel: p.rawChannel, label: inputName0(p)) }
+        inputs.collect { InParam p -> new ChannelHandler(channel: p.rawChannel, label: inputName0(p)) }
 
     }
 
diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/ForwardClosure.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/ForwardClosure.groovy
deleted file mode 100644
index 8fb48e4515..0000000000
--- a/modules/nextflow/src/main/groovy/nextflow/processor/ForwardClosure.groovy
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Copyright 2013-2023, Seqera Labs
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package nextflow.processor
-
-import groovy.transform.CompileStatic
-import groovyx.gpars.dataflow.operator.DataflowProcessor
-
-/**
- * Implements the closure which *combines* all the iteration
- *
- * @param numOfInputs Number of in/out channel
- * @param indexes The list of indexes which identify the position of iterators in the input channels
- * @return The closure implementing the iteration/forwarding logic
- */
-@CompileStatic
-class ForwardClosure extends Closure {
-
-    final private Integer len
-
-    final private int numOfParams
-
-    final private List indexes
-
-    ForwardClosure(int len, List indexes) {
-        super(null, null);
-        this.len = len
-        this.numOfParams = len+1
-        this.indexes = indexes
-    }
-
-    @Override
-    int getMaximumNumberOfParameters() {
-        numOfParams
-    }
-
-    @Override
-    Class[] getParameterTypes() {
-        def result = new Class[numOfParams]
-        for( int i=0; i cmb = itr.combinations()
-
-        for( int i=0; i iteratorIndexes = []
-        config.getInputs().eachWithIndex { param, index ->
-            if( param instanceof EachInParam ) {
-                log.trace "Process ${name} > got each param: ${param.name} at index: ${index} -- ${param.dump()}"
-                iteratorIndexes << index
-            }
-        }
-
-        /**
-         * The thread pool used by GPars. The thread pool to be used is set in the static
-         * initializer of {@link nextflow.cli.CmdRun} class. See also {@link nextflow.util.CustomPoolFactory}
-         */
-        final PGroup group = Dataflow.retrieveCurrentDFPGroup()
-
-        /*
-         * When one (or more) {@code each} are declared as input, it is created an extra
-         * operator which will receive the inputs from the channel (excepts the values over iterate)
-         *
-         * The operator will *expand* the received inputs, iterating over the user provided value and
-         * forwarding the final values the the second *parallel* processor executing the user specified task
-         */
-        if( iteratorIndexes ) {
-            log.debug "Creating *combiner* operator for each param(s) at index(es): ${iteratorIndexes}"
-
-            // don't care about the last channel, being the control channel it doesn't bring real values
-            final size = opInputs.size()-1
-
-            // the iterator operator needs to executed just one time
-            // thus add a dataflow queue binding a single value and then a stop signal
-            def termination = new DataflowQueue<>()
-            termination << Boolean.TRUE
-            opInputs[size] = termination
-
-            // the channel forwarding the data from the *iterator* process to the target task
-            final linkingChannels = new ArrayList(size)
-            size.times { linkingChannels[it] = new DataflowQueue() }
+        // determine whether the process is executed only once
+        this.singleton = source == null || !CH.isChannelQueue(source)
+        config.getOutputs().setSingleton(singleton)
 
-            // the script implementing the iterating process
-            final forwarder = new ForwardClosure(size, iteratorIndexes)
+        // create inputs with control channel
+        final control = CH.queue()
+        control.bind(Boolean.TRUE)
 
-            // instantiate the iteration process
-            def DataflowOperator op1
-            def stopAfterFirstRun = allScalarValues
-            def interceptor = new BaseProcessInterceptor(opInputs, stopAfterFirstRun)
-            def params = [inputs: opInputs, outputs: linkingChannels, maxForks: 1, listeners: [interceptor]]
-            session.allOperators << (op1 = new DataflowOperator(group, params, forwarder))
-            // fix issue #41
-            start(op1)
+        final opInputs = source != null ? [source, control] : [control]
 
-            // set as next inputs the result channels of the iteration process
-            // adding the 'control' channel removed previously
-            opInputs = new ArrayList(size+1)
-            opInputs.addAll( linkingChannels )
-            opInputs.add( config.getInputs().getChannels().last() )
-        }
-
-        /*
-         * finally create the operator
-         */
         // note: do not specify the output channels in the operator declaration
         // this allows us to manage them independently from the operator life-cycle
-        this.singleton = allScalarValues && !hasEachParams
-        this.openPorts = createPortsArray(opInputs.size())
-        config.getOutputs().setSingleton(singleton)
-        def interceptor = new TaskProcessorInterceptor(opInputs, singleton)
+        def interceptor = new TaskProcessorInterceptor(source, control, singleton)
         def params = [inputs: opInputs, maxForks: session.poolSize, listeners: [interceptor] ]
         def invoke = new InvokeTaskAdapter(this, opInputs.size())
-        session.allOperators << (operator = new DataflowOperator(group, params, invoke))
+
+        // note: the GPars thread pool is set in the static initializer of {@link nextflow.cli.CmdRun}
+        // see also {@link nextflow.util.CustomPoolFactory}
+        this.operator = new DataflowOperator( Dataflow.retrieveCurrentDFPGroup(), params, invoke )
 
         // notify the creation of a new vertex the execution DAG
         NodeMarker.addProcessNode(this, config.getInputs(), config.getOutputs())
 
-        // fix issue #41
-        start(operator)
-    }
-
-    private start(DataflowProcessor op) {
-        if( !NF.dsl2 ) {
-            op.start()
-            return
-        }
+        // start the operator
+        session.allOperators << operator
         session.addIgniter {
             log.debug "Starting process > $name"
-            op.start()
+            operator.start()
         }
     }
 
-    private AtomicIntegerArray createPortsArray(int size) {
-        def result = new AtomicIntegerArray(size)
-        for( int i=0; i $name with params=$params; values=$values"
@@ -2375,69 +2277,23 @@ class TaskProcessor {
 
         def statusStr = !completed && !terminated ? 'status=ACTIVE' : ( completed && terminated ? 'status=TERMINATED' : "completed=$completed; terminated=$terminated" )
         result << "  $statusStr\n"
-        // add extra info about port statuses
-        for( int i=0; i inputs
-
-        final boolean stopAfterFirstRun
+    class TaskProcessorInterceptor extends DataflowEventAdapter {
 
-        final int len
+        final DataflowReadChannel source
 
         final DataflowQueue control
 
-        final int first
-
-        BaseProcessInterceptor( List inputs, boolean stop ) {
-            this.inputs = new ArrayList<>(inputs)
-            this.stopAfterFirstRun = stop
-            this.len = inputs.size()
-            this.control = (DataflowQueue)inputs.get(len-1)
-            this.first = inputs.findIndexOf { CH.isChannelQueue(it) }
-        }
-
-        @Override
-        Object messageArrived(final DataflowProcessor processor, final DataflowReadChannel channel, final int index, final Object message) {
-            if( len == 1 || stopAfterFirstRun ) {
-                // -- kill itself
-                control.bind(PoisonPill.instance)
-            }
-            else if( index == first ) {
-                // the `if` condition guarantees only and only one signal message (the true value)
-                // is bound to the control message for a complete set of input values delivered
-                // to the process -- the control message is need to keep the process running
-                control.bind(Boolean.TRUE)
-            }
-
-            return message
-        }
-    }
+        final boolean singleton
 
-    /**
-     *  Intercept dataflow process events
-     */
-    class TaskProcessorInterceptor extends BaseProcessInterceptor {
-
-        TaskProcessorInterceptor(List inputs, boolean stop) {
-            super(inputs, stop)
+        TaskProcessorInterceptor(DataflowReadChannel source, DataflowQueue control, boolean singleton) {
+            this.source = source
+            this.control = control
+            this.singleton = singleton
         }
 
         @Override
@@ -2450,13 +2306,13 @@ class TaskProcessor {
             // task index must be created here to guarantee consistent ordering
             // with the sequence of messages arrival since this method is executed in a thread safe manner
             final params = new TaskStartParams(TaskId.next(), indexCount.incrementAndGet())
+            final args = source != null ? messages.first() : []
             final result = new ArrayList(2)
             result[0] = params
-            result[1] = messages
+            result[1] = args
             return result
         }
 
-
         @Override
         void afterRun(DataflowProcessor processor, List messages) {
             // apparently auto if-guard instrumented by @Slf4j is not honoured in inner classes - add it explicitly
@@ -2467,23 +2323,24 @@ class TaskProcessor {
 
         @Override
         Object messageArrived(final DataflowProcessor processor, final DataflowReadChannel channel, final int index, final Object message) {
-            // apparently auto if-guard instrumented by @Slf4j is not honoured in inner classes - add it explicitly
-            if( log.isTraceEnabled() ) {
-                def channelName = config.getInputs()?.names?.get(index)
-                def taskName = currentTask.get()?.name ?: name
-                log.trace "<${taskName}> Message arrived -- ${channelName} => ${message}"
+            if( singleton ) {
+                // -- kill the process
+                control.bind(PoisonPill.instance)
+            }
+            else {
+                // -- send a control message for each new source item to keep the process running
+                control.bind(Boolean.TRUE)
             }
 
-            super.messageArrived(processor, channel, index, message)
+            return message
         }
 
         @Override
         Object controlMessageArrived(final DataflowProcessor processor, final DataflowReadChannel channel, final int index, final Object message) {
             // apparently auto if-guard instrumented by @Slf4j is not honoured in inner classes - add it explicitly
             if( log.isTraceEnabled() ) {
-                def channelName = config.getInputs()?.names?.get(index)
                 def taskName = currentTask.get()?.name ?: name
-                log.trace "<${taskName}> Control message arrived ${channelName} => ${message}"
+                log.trace "<${taskName}> Control message arrived => ${message}"
             }
 
             super.controlMessageArrived(processor, channel, index, message)
@@ -2492,7 +2349,7 @@ class TaskProcessor {
                 // apparently auto if-guard instrumented by @Slf4j is not honoured in inner classes - add it explicitly
                 if( log.isTraceEnabled() )
                     log.trace "<${name}> Poison pill arrived; port: $index"
-                openPorts.set(index, 0) // mark the port as closed
+                closed.set(true)
                 state.update { StateObj it -> it.poison() }
             }
 
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessConfig.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessConfig.groovy
index ffdaa173c8..9f7c74653f 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessConfig.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessConfig.groovy
@@ -24,7 +24,6 @@ import nextflow.executor.BashWrapperBuilder
 import nextflow.processor.ErrorStrategy
 import nextflow.processor.TaskConfig
 import static nextflow.util.CacheHelper.HashMode
-import nextflow.script.params.DefaultInParam
 import nextflow.script.params.DefaultOutParam
 import nextflow.script.params.InputsList
 import nextflow.script.params.OutputsList
@@ -173,14 +172,6 @@ class ProcessConfig implements Map, Cloneable {
         outputs
     }
 
-    /**
-     * Defines a special *dummy* input parameter, when no inputs are
-     * provided by the user for the current task
-     */
-    void fakeInput() {
-        new DefaultInParam(this)
-    }
-
     void fakeOutput() {
         new DefaultOutParam(this)
     }
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy
index d68e9625d7..56390cb9a1 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy
@@ -25,6 +25,7 @@ import nextflow.Global
 import nextflow.Session
 import nextflow.exception.ScriptRuntimeException
 import nextflow.extension.CH
+import nextflow.extension.CombineOp
 import nextflow.extension.MergeOp
 import nextflow.script.dsl.ProcessBuilder
 import nextflow.script.params.BaseInParam
@@ -69,7 +70,7 @@ class ProcessDef extends BindableDef implements IterableDef, ChainableDef {
     /**
      * The resolved process configuration
      */
-    private ProcessConfig processConfig
+    private ProcessConfig config
 
     /**
      * The actual process implementation
@@ -87,7 +88,7 @@ class ProcessDef extends BindableDef implements IterableDef, ChainableDef {
         this.processName = name
         this.baseName = name
         this.taskBody = body
-        this.processConfig = config
+        this.config = config
     }
 
     static String stripScope(String str) {
@@ -96,14 +97,14 @@ class ProcessDef extends BindableDef implements IterableDef, ChainableDef {
 
     protected void initialize() {
         // apply config settings to the process
-        new ProcessBuilder(processConfig).applyConfig((Map)session.config.process, baseName, simpleName, processName)
+        new ProcessBuilder(config).applyConfig((Map)session.config.process, baseName, simpleName, processName)
     }
 
     @Override
     ProcessDef clone() {
         def result = (ProcessDef)super.clone()
         result.@taskBody = taskBody.clone()
-        result.@processConfig = processConfig.clone()
+        result.@config = config.clone()
         return result
     }
 
@@ -113,13 +114,13 @@ class ProcessDef extends BindableDef implements IterableDef, ChainableDef {
         def result = clone()
         result.@processName = name
         result.@simpleName = stripScope(name)
-        result.@processConfig.processName = name
+        result.@config.processName = name
         return result
     }
 
-    private InputsList getDeclaredInputs() { processConfig.getInputs() }
+    private InputsList getDeclaredInputs() { config.getInputs() }
 
-    private OutputsList getDeclaredOutputs() { processConfig.getOutputs() }
+    private OutputsList getDeclaredOutputs() { config.getOutputs() }
 
     BaseScript getOwner() { owner }
 
@@ -129,7 +130,7 @@ class ProcessDef extends BindableDef implements IterableDef, ChainableDef {
 
     String getBaseName() { baseName }
 
-    ProcessConfig getProcessConfig() { processConfig }
+    ProcessConfig getProcessConfig() { config }
 
     ChannelOut getOut() {
         if( output==null )
@@ -152,11 +153,7 @@ class ProcessDef extends BindableDef implements IterableDef, ChainableDef {
         initialize()
 
         // create merged input channel
-        final inputs = ChannelOut.spread(args).collect(ch -> getInChannel(ch))
-        if( inputs.findAll(ch -> CH.isChannelQueue(ch)).size() > 1 )
-            throw new ScriptRuntimeException("Process `$name` received multiple queue channel inputs which is not allowed")
-
-        final input = CH.getReadChannel(new MergeOp(inputs.first(), inputs[1..0, "Process output should contains at least one channel"
         return output = new ChannelOut(copyOuts)
     }
 
+    private DataflowReadChannel getSourceChannel(Object[] args) {
+        if( args.length == 0 )
+            return null
+
+        // normalize and validate input channels
+        final inputs = ChannelOut.spread(args).collect(ch -> getInChannel(ch))
+        if( inputs.findAll(ch -> CH.isChannelQueue(ch)).size() > 1 )
+            throw new ScriptRuntimeException("Process `$name` received multiple queue channel inputs which is not allowed")
+
+        // merge input channels
+        // TODO: skip `each` inputs
+        def source = CH.getReadChannel(new MergeOp(inputs.first(), inputs[1.. each param '${param.name}' at index ${i} -- ${param.dump()}"
+            source = CH.getReadChannel(new CombineOp(source, param.getInChannel()))
+        }
+
+        return source
+    }
+
     private DataflowReadChannel getInChannel(Object obj) {
         if( obj == null )
             throw new IllegalArgumentException('A process input channel evaluates to null')
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessBuilder.groovy b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessBuilder.groovy
index aded6a3944..6e08055743 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessBuilder.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessBuilder.groovy
@@ -443,6 +443,8 @@ class ProcessBuilder {
     }
 
     InParam _in_tuple( Object... obj ) {
+        if( obj.length < 2 )
+            throw new IllegalArgumentException("Input `tuple` must define at least two elements -- Check process `$processName`")
         new TupleInParam(config).bind(obj)
     }
 
@@ -501,11 +503,15 @@ class ProcessBuilder {
     }
 
     OutParam _out_tuple( Object... obj ) {
+        if( obj.length < 2 )
+            throw new IllegalArgumentException("Output `tuple` must define at least two elements -- Check process `$processName`")
         new TupleOutParam(config)
                 .bind(obj)
     }
 
     OutParam _out_tuple( Map opts, Object... obj ) {
+        if( obj.length < 2 )
+            throw new IllegalArgumentException("Output `tuple` must define at least two elements -- Check process `$processName`")
         new TupleOutParam(config)
                 .setOptions(opts)
                 .bind(obj)
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/BaseInParam.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/BaseInParam.groovy
index a932af4863..43c2ae70da 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/params/BaseInParam.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/params/BaseInParam.groovy
@@ -75,11 +75,6 @@ abstract class BaseInParam extends BaseParam implements InParam {
     protected DataflowReadChannel inputValToChannel( value ) {
         checkFromNotNull(value)
 
-        if( this instanceof DefaultInParam ) {
-            assert value instanceof DataflowQueue
-            return value
-        }
-
         if ( value instanceof DataflowReadChannel || value instanceof DataflowBroadcast )  {
             return CH.getReadChannel(value)
         }
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/DefaultInParam.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/DefaultInParam.groovy
deleted file mode 100644
index 80b85ad311..0000000000
--- a/modules/nextflow/src/main/groovy/nextflow/script/params/DefaultInParam.groovy
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Copyright 2013-2023, Seqera Labs
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package nextflow.script.params
-
-import nextflow.extension.CH
-import nextflow.script.ProcessConfig
-/**
- * Model a process default input parameter
- *
- * @author Paolo Di Tommaso 
- */
-final class DefaultInParam extends ValueInParam {
-
-    @Override
-    String getTypeName() { 'default' }
-
-    DefaultInParam(ProcessConfig config) {
-        super(config)
-        // This must be a dataflow queue channel to which
-        // just a value is bound -- No STOP value has to be emitted
-        // because this channel is used to control to process termination
-        // See TaskProcessor.BaseProcessInterceptor#messageArrived
-        final channel = CH.queue()
-        channel.bind(Boolean.TRUE)
-        setFrom(channel)
-        bind('$')
-    }
-}

From 041e10aa808230d0f398e0854137cc7a6c515b3e Mon Sep 17 00:00:00 2001
From: Ben Sherman 
Date: Fri, 1 Dec 2023 17:40:18 -0600
Subject: [PATCH 08/36] Remove params from WorkflowFn

Signed-off-by: Ben Sherman 
---
 .../groovy/nextflow/ast/WorkflowFn.groovy     |  1 -
 .../nextflow/ast/WorkflowFnXform.groovy       | 12 --------
 .../groovy/nextflow/script/BaseScript.groovy  |  4 ---
 .../groovy/nextflow/script/WorkflowDef.groovy | 28 +++++++++----------
 .../script/dsl/WorkflowBuilder.groovy         |  5 ----
 5 files changed, 13 insertions(+), 37 deletions(-)

diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/WorkflowFn.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/WorkflowFn.groovy
index ca9ef36f4c..77dfcce8e2 100644
--- a/modules/nextflow/src/main/groovy/nextflow/ast/WorkflowFn.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/ast/WorkflowFn.groovy
@@ -32,6 +32,5 @@ import java.lang.annotation.Target
     boolean main() default false
 
     // injected via AST transform
-    Class params()
     String source()
 }
diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/WorkflowFnXform.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/WorkflowFnXform.groovy
index f57d339dd2..94406d5885 100644
--- a/modules/nextflow/src/main/groovy/nextflow/ast/WorkflowFnXform.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/ast/WorkflowFnXform.groovy
@@ -24,10 +24,6 @@ import org.codehaus.groovy.ast.ASTNode
 import org.codehaus.groovy.ast.AnnotationNode
 import org.codehaus.groovy.ast.ClassCodeVisitorSupport
 import org.codehaus.groovy.ast.MethodNode
-import org.codehaus.groovy.ast.Parameter
-import org.codehaus.groovy.ast.expr.Expression
-import org.codehaus.groovy.ast.expr.ListExpression
-import org.codehaus.groovy.ast.stmt.ExpressionStatement
 import org.codehaus.groovy.control.SourceUnit
 /**
  * Implements syntax transformations for workflow functions.
@@ -57,14 +53,6 @@ class WorkflowFnXform extends ClassCodeVisitorSupport {
     }
 
     protected void transform(MethodNode method, AnnotationNode annotation) {
-        // append method params
-        final params = method.getParameters() as List
-        annotation.addMember( 'params', closureX( block( new ExpressionStatement(
-            new ListExpression(
-                params.collect(p -> (Expression)constX(p.getName()))
-            )
-        ) ) ) )
-
         // append workflow source
         annotation.addMember( 'source', constX( getSource(method.getCode()) ) )
     }
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy b/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy
index 64170181f2..5e49bc3394 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy
@@ -239,10 +239,6 @@ abstract class BaseScript extends Script implements ExecutionContext {
             ? new WorkflowBuilder(this)
             : new WorkflowBuilder(this, name)
 
-        // get method parameters
-        final params = (List)((Closure)workflowFn.params().newInstance(this, this)).call()
-        builder.withParams(params)
-
         // create body
         final body = new BodyDef( this.&"${name}", workflowFn.source(), 'workflow', [] )
         builder.withBody(body)
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/WorkflowDef.groovy b/modules/nextflow/src/main/groovy/nextflow/script/WorkflowDef.groovy
index 7a294a17b3..de7cde7351 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/WorkflowDef.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/WorkflowDef.groovy
@@ -191,20 +191,9 @@ class WorkflowDef extends BindableDef implements ChainableDef, IterableDef, Exec
     }
 
     private Object run0(Object[] args) {
-        collectInputs(binding, args)
-        // invoke the workflow execution
-        final result = run1(body.closure, args)
-
-        // apply return value to declared outputs, binding
-        normalizeOutput(result)
-
-        // collect the workflow outputs
-        output = collectOutputs(declaredOutputs)
-        return output
-    }
-
-    private Object run1(Closure closure, Object[] args) {
+        final closure = body.closure
         if( closure instanceof MethodClosure ) {
+            // invoke the workflow function with args
             final target = closure.owner
             final name = closure.method
             final meta = target.metaClass.getMetaMethod(name, args)
@@ -213,13 +202,22 @@ class WorkflowDef extends BindableDef implements ChainableDef, IterableDef, Exec
             final method = target.getClass().getMethod(name, meta.getNativeParameterTypes())
             if( method == null )
                 throw new MissingMethodException(name, target.getClass(), args)
-            return method.invoke(target, args)
+            final result = method.invoke(target, args)
+
+            // apply return value to declared outputs, binding
+            normalizeOutput(result)
         }
         else {
+            // invoke the workflow closure with delegate
+            collectInputs(binding, args)
             closure.setDelegate(binding)
             closure.setResolveStrategy(Closure.DELEGATE_FIRST)
-            return closure.call()
+            closure.call()
         }
+
+        // collect the workflow outputs
+        output = collectOutputs(declaredOutputs)
+        return output
     }
 
     private void normalizeOutput(Object result) {
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/dsl/WorkflowBuilder.groovy b/modules/nextflow/src/main/groovy/nextflow/script/dsl/WorkflowBuilder.groovy
index 506a2895e3..2e9e1497d5 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/dsl/WorkflowBuilder.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/dsl/WorkflowBuilder.groovy
@@ -56,11 +56,6 @@ class WorkflowBuilder {
             throw new MissingMethodException(name, WorkflowDef, args)
     }
 
-    WorkflowBuilder withParams(List params) {
-        for( String param : params )
-            takes.put(param, true)
-    }
-
     WorkflowBuilder withBody(BodyDef body) {
         this.body = body
         return this

From a52a829b04c56492536017d4db6c628e17749c5b Mon Sep 17 00:00:00 2001
From: Ben Sherman 
Date: Fri, 1 Dec 2023 18:40:34 -0600
Subject: [PATCH 09/36] Simplify ProcessFn param names

Signed-off-by: Ben Sherman 
---
 .../main/groovy/nextflow/ast/ProcessFn.groovy |  2 +-
 .../groovy/nextflow/ast/ProcessFnXform.groovy |  8 ++----
 .../nextflow/processor/TaskProcessor.groovy   | 24 ++++------------
 .../groovy/nextflow/processor/TaskRun.groovy  | 28 +++++++++++--------
 .../groovy/nextflow/script/BaseScript.groovy  |  6 +---
 .../nextflow/script/ProcessConfig.groovy      | 13 ++-------
 .../nextflow/script/dsl/ProcessBuilder.groovy |  5 ++++
 7 files changed, 35 insertions(+), 51 deletions(-)

diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/ProcessFn.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/ProcessFn.groovy
index 203118b061..ca268dc668 100644
--- a/modules/nextflow/src/main/groovy/nextflow/ast/ProcessFn.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/ast/ProcessFn.groovy
@@ -37,6 +37,6 @@ import java.lang.annotation.Target
     boolean shell() default false
 
     // injected via AST transform
-    Class params()
+    String[] params()
     String source()
 }
diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/ProcessFnXform.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/ProcessFnXform.groovy
index 6a04d1af07..dcc8feee51 100644
--- a/modules/nextflow/src/main/groovy/nextflow/ast/ProcessFnXform.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/ast/ProcessFnXform.groovy
@@ -99,11 +99,9 @@ class ProcessFnXform extends ClassCodeVisitorSupport {
         // TODO: append stub source
 
         // append method params
-        annotation.addMember( 'params', closureX( block( new ExpressionStatement(
-            new ListExpression(
-                params.collect(p -> (Expression)constX(p.getName()))
-            )
-        ) ) ) )
+        annotation.addMember( 'params', new ListExpression(
+            params.collect(p -> (Expression)constX(p.getName()))
+        ) )
 
         // append script source
         annotation.addMember( 'source', constX( getSource(method.getCode()) ) )
diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy
index 1008eb192b..b7333fca32 100644
--- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy
@@ -66,7 +66,6 @@ import nextflow.exception.ProcessFailedException
 import nextflow.exception.ProcessRetryableException
 import nextflow.exception.ProcessSubmitTimeoutException
 import nextflow.exception.ProcessUnrecoverableException
-import nextflow.exception.ScriptRuntimeException
 import nextflow.exception.ShowOnlyExceptionMessage
 import nextflow.exception.UnexpectedException
 import nextflow.executor.CachedTaskHandler
@@ -458,7 +457,7 @@ class TaskProcessor {
 
         // note: the GPars thread pool is set in the static initializer of {@link nextflow.cli.CmdRun}
         // see also {@link nextflow.util.CustomPoolFactory}
-        this.operator = new DataflowOperator( Dataflow.retrieveCurrentDFPGroup(), params, invoke )
+        this.operator = new DataflowOperator(Dataflow.retrieveCurrentDFPGroup(), params, invoke)
 
         // notify the creation of a new vertex the execution DAG
         NodeMarker.addProcessNode(this, config.getInputs(), config.getOutputs())
@@ -492,23 +491,12 @@ class TaskProcessor {
         // -- set the task instance as the current in this thread
         currentTask.set(task)
 
-        // -- add task config to arguments
+        // -- prepend task config to arguments (for process function)
         values.push(task.config)
 
-        // -- validate task arguments
-        if( config.params.size() != values.size() )
-            throw new ScriptRuntimeException("Process $name expected ${config.params.size()} arguments but received ${values.size()}: ${values}")
-
-        for( int i = 0; i < config.params.size(); i++ ) {
-            final param = config.params[i]
-            final value = values[i]
-            if( !param.type.isAssignableFrom(value.class) )
-                log.warn1 "Process $name > expected type ${param.type.name} for param ${param.name} but got a ${value.class.name}"
-        }
-
-        // -- add arguments to task context
-        for( int i = 1; i < config.params.size(); i++ )
-            task.context.put(config.params[i].name, values[i])
+        // -- add arguments to task context (for process function)
+        for( int i = 1; i < config.params.length; i++ )
+            task.context.put(config.params[i], values[i])
 
         // -- validate input lengths
         validateInputTuples(values)
@@ -528,7 +516,7 @@ class TaskProcessor {
         }
         else {
             // -- resolve the task command script
-            task.resolve(taskBody, config.params*.name)
+            task.resolve(taskBody, values.toArray())
         }
 
         // -- verify if exists a stored result for this case,
diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy
index 54e00dbf4e..778df8b419 100644
--- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy
@@ -47,6 +47,7 @@ import nextflow.script.params.OutParam
 import nextflow.script.params.StdInParam
 import nextflow.script.params.ValueOutParam
 import nextflow.spack.SpackCache
+import org.codehaus.groovy.runtime.MethodClosure
 /**
  * Models a task instance
  *
@@ -711,33 +712,36 @@ class TaskRun implements Cloneable {
      * 3) assign the `script` code to execute
      *
      * @param body
-     * @param params
+     * @param args
      */
-    @PackageScope void resolve(BodyDef body, List params=[]) {
+    @PackageScope void resolve(BodyDef body, Object[] args) {
 
         // -- initialize the task code to be executed
-        this.code = body.closure.clone() as Closure
-        this.code.delegate = this.context
-        this.code.setResolveStrategy(Closure.DELEGATE_ONLY)
+        this.code = body.closure
+
+        // -- provide arguments directly or via delegate
+        if( code instanceof MethodClosure ) {
+            code = code.curry(args)
+        }
+        else {
+            code = code.clone() as Closure
+            code.setDelegate(this.context)
+            code.setResolveStrategy(Closure.DELEGATE_ONLY)
+        }
 
         // -- set the task source
         this.body = body
         // note: this may be overwritten when a template file is used
         this.source = body.source
 
-        // -- collect method args from task context
-        final args = params.collect(param -> context[param]).toArray()
-
-        if( body.type != ScriptType.SCRIPTLET ) {
-            code = code.curry(args)
+        if( body.type != ScriptType.SCRIPTLET )
             return
-        }
 
         // Important!
         // when the task is implemented by a script string
         // Invoke the closure which returns the script with all the variables replaced with the actual values
         try {
-            final result = code.call(args)
+            final result = code.call()
             if ( result instanceof Path ) {
                 script = renderTemplate(result, body.isShell)
             }
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy b/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy
index 5e49bc3394..0c4b7b8918 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy
@@ -207,11 +207,7 @@ abstract class BaseScript extends Script implements ExecutionContext {
         applyDsl(builder, processFn.outputs())
 
         // get method parameters
-        final paramNames = (List)((Closure)processFn.params().newInstance(this, this)).call()
-        final params = (0 ..< paramNames.size()).collect( i ->
-            new Parameter( paramNames[i], method.getParameters()[i].getType() )
-        )
-        builder.config.params = params
+        builder.withParams(processFn.params())
 
         // determine process type
         def type
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessConfig.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessConfig.groovy
index 9f7c74653f..40e21830a6 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessConfig.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessConfig.groovy
@@ -16,7 +16,6 @@
 
 package nextflow.script
 
-import groovy.transform.Immutable
 import groovy.transform.PackageScope
 import groovy.util.logging.Slf4j
 import nextflow.Const
@@ -67,9 +66,9 @@ class ProcessConfig implements Map, Cloneable {
     private String processName
 
     /**
-     * List of parameters defined by a process function.
+     * List of parameter names defined by a process function.
      */
-    List params
+    String[] params
 
     /**
      * List of process input definitions
@@ -160,7 +159,7 @@ class ProcessConfig implements Map, Cloneable {
         return new TaskConfig(configProperties)
     }
 
-    List getParams() {
+    String[] getParams() {
         params
     }
 
@@ -220,9 +219,3 @@ class ProcessConfig implements Map, Cloneable {
     }
 
 }
-
-@Immutable
-class Parameter {
-    String name
-    Class type
-}
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessBuilder.groovy b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessBuilder.groovy
index 6e08055743..c4721f5653 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessBuilder.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessBuilder.groovy
@@ -530,6 +530,11 @@ class ProcessBuilder {
 
     /// SCRIPT
 
+    ProcessBuilder withParams(String[] params) {
+        config.params = params
+        return this
+    }
+
     ProcessBuilder withBody(BodyDef body) {
         this.body = body
         return this

From 570892c3d0b6d56c8b8c4f3e9169abe4cd7fa353 Mon Sep 17 00:00:00 2001
From: Ben Sherman 
Date: Sat, 2 Dec 2023 12:24:33 -0600
Subject: [PATCH 10/36] Separate `InParam`s from task config

Signed-off-by: Ben Sherman 
---
 .../src/main/groovy/nextflow/dag/DAG.groovy   |   4 +-
 .../PathArityAware.groovy}                    |   4 +-
 .../nextflow/processor/TaskConfig.groovy      |  20 ++--
 .../nextflow/processor/TaskFileInput.groovy   |  76 +++++++++++++
 .../nextflow/processor/TaskProcessor.groovy   | 105 +++++++-----------
 .../groovy/nextflow/processor/TaskRun.groovy  |  71 +++---------
 .../nextflow/script/ProcessConfig.groovy      |  13 ++-
 .../script/dsl/ProcessInputsBuilder.groovy    |  25 ++---
 .../nextflow/script/params/FileInParam.groovy |  81 +-------------
 .../script/params/FileOutParam.groovy         |   3 +-
 10 files changed, 177 insertions(+), 225 deletions(-)
 rename modules/nextflow/src/main/groovy/nextflow/{script/params/ArityParam.groovy => processor/PathArityAware.groovy} (98%)
 create mode 100644 modules/nextflow/src/main/groovy/nextflow/processor/TaskFileInput.groovy

diff --git a/modules/nextflow/src/main/groovy/nextflow/dag/DAG.groovy b/modules/nextflow/src/main/groovy/nextflow/dag/DAG.groovy
index bc92a3b3ce..323018e71c 100644
--- a/modules/nextflow/src/main/groovy/nextflow/dag/DAG.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/dag/DAG.groovy
@@ -97,7 +97,7 @@ class DAG {
      */
     void addProcessNode( String label, InputsList inputs, OutputsList outputs, TaskProcessor process=null ) {
         assert label
-        assert inputs
+        assert inputs != null
         assert outputs
         addVertex( Type.PROCESS, label, normalizeInputs(inputs), normalizeOutputs(outputs), process )
     }
@@ -111,7 +111,7 @@ class DAG {
      */
     void addOperatorNode( String label, inputs, outputs, List operators=null )  {
         assert label
-        assert inputs
+        assert inputs != null
         addVertex(Type.OPERATOR, label, normalizeChannels(inputs), normalizeChannels(outputs), operators )
     }
 
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/ArityParam.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/PathArityAware.groovy
similarity index 98%
rename from modules/nextflow/src/main/groovy/nextflow/script/params/ArityParam.groovy
rename to modules/nextflow/src/main/groovy/nextflow/processor/PathArityAware.groovy
index 3c1a425288..59c1b5bfac 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/params/ArityParam.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/processor/PathArityAware.groovy
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package nextflow.script.params
+package nextflow.processor
 
 import groovy.transform.CompileStatic
 import groovy.transform.EqualsAndHashCode
@@ -26,7 +26,7 @@ import nextflow.exception.IllegalArityException
  * @author Ben Sherman 
  */
 @CompileStatic
-trait ArityParam {
+trait PathArityAware {
 
     Range arity
 
diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskConfig.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskConfig.groovy
index e7f0658148..512b76ed0e 100644
--- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskConfig.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskConfig.groovy
@@ -16,13 +16,12 @@
 
 package nextflow.processor
 
-import nextflow.util.CmdLineOptionMap
-
 import static nextflow.processor.TaskProcessor.*
 
 import java.nio.file.Path
 
 import groovy.transform.CompileStatic
+import groovy.transform.PackageScope
 import nextflow.Const
 import nextflow.ast.DslCodeVisitor
 import nextflow.exception.AbortOperationException
@@ -33,6 +32,7 @@ import nextflow.executor.res.DiskResource
 import nextflow.k8s.model.PodOptions
 import nextflow.script.TaskClosure
 import nextflow.util.CmdLineHelper
+import nextflow.util.CmdLineOptionMap
 import nextflow.util.Duration
 import nextflow.util.MemoryUnit
 /**
@@ -45,6 +45,9 @@ class TaskConfig extends LazyMap implements Cloneable {
 
     static public final int EXIT_ZERO = 0
 
+    @PackageScope
+    static final List LAZY_MAP_PROPERTIES = ['env', 'ext']
+
     private transient Map cache = new LinkedHashMap(20)
 
     TaskConfig() {  }
@@ -78,9 +81,10 @@ class TaskConfig extends LazyMap implements Cloneable {
         // clear cache to force re-compute dynamic entries
         this.cache.clear()
 
-        // set the binding context for 'ext' map
-        if( target.ext instanceof LazyMap )
-            (target.ext as LazyMap).binding = context
+        // set the binding context for lazy map properties
+        for( def key in LAZY_MAP_PROPERTIES )
+            if( target.get(key) instanceof LazyMap )
+                (target.get(key) as LazyMap).binding = context
 
         // set the this object in the task context in order to allow task properties to be resolved in process script
         context.put(TASK_CONTEXT_PROPERTY_NAME, this)
@@ -140,7 +144,7 @@ class TaskConfig extends LazyMap implements Cloneable {
             return cache.get(key)
 
         def result
-        if( key == 'ext' ) {
+        if( key in LAZY_MAP_PROPERTIES ) {
             if( target.containsKey(key) )
                 result = target.get(key)
             else {
@@ -168,7 +172,7 @@ class TaskConfig extends LazyMap implements Cloneable {
             }
             target.put(key, value)
         }
-        else if( key == 'ext' && value instanceof Map ) {
+        else if( key in LAZY_MAP_PROPERTIES && value instanceof Map ) {
             super.put( key, new LazyMap(value) )
         }
         else {
@@ -586,7 +590,7 @@ class LazyMap implements Map {
          * note: 'ext' property is meant for extension attributes
          * as it should be preserved as LazyMap
          */
-        else if( value instanceof Map && name!='ext' ) {
+        else if( value instanceof Map && name !in TaskConfig.LAZY_MAP_PROPERTIES ) {
             return resolveParams(name, value)
         }
 
diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskFileInput.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskFileInput.groovy
new file mode 100644
index 0000000000..b93064ae0e
--- /dev/null
+++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskFileInput.groovy
@@ -0,0 +1,76 @@
+/*
+ * Copyright 2013-2023, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package nextflow.processor
+
+import groovy.transform.CompileStatic
+
+/**
+ * Models a file input directive, which defines the file
+ * or set of files to be staged into the task environment.
+ *
+ * @author Ben Sherman 
+ */
+@CompileStatic
+class TaskFileInput implements PathArityAware {
+    private Object value
+    private boolean coerceToPath
+    private String name
+    private Object filePattern
+
+    TaskFileInput(Object value, boolean coerceToPath, String name, Map opts) {
+        this.value = value
+        this.coerceToPath = coerceToPath
+        this.name = name
+        this.filePattern = opts.stageAs ?: opts.name
+
+        if( opts.arity )
+            this.setArity(opts.arity.toString())
+    }
+
+    Object getValue(Map ctx) {
+        return resolve(ctx, value)
+    }
+
+    boolean isPathQualifier() {
+        return coerceToPath
+    }
+
+    String getName() {
+        return name
+    }
+
+    String getFilePattern(Map ctx) {
+        if( filePattern != null )
+            return resolve(ctx, filePattern)
+
+        if( value != null )
+            return resolve(ctx, value)
+
+        return filePattern = '*'
+    }
+
+    private Object resolve( Map ctx, value ) {
+        if( value instanceof GString )
+            return value.cloneAsLazy(ctx)
+
+        if( value instanceof Closure )
+            return ctx.with(value)
+
+        return value
+    }
+
+}
diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy
index b7333fca32..c438e8e7a3 100644
--- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy
@@ -491,20 +491,16 @@ class TaskProcessor {
         // -- set the task instance as the current in this thread
         currentTask.set(task)
 
-        // -- prepend task config to arguments (for process function)
-        values.push(task.config)
-
         // -- add arguments to task context (for process function)
         for( int i = 1; i < config.params.length; i++ )
-            task.context.put(config.params[i], values[i])
+            task.context.put(config.params[i], values[i-1])
 
         // -- validate input lengths
         validateInputTuples(values)
 
         // -- map the inputs to a map and use to delegate closure values interpolation
-        final secondPass = [:]
-        int count = makeTaskContextStage1(task, secondPass, values)
-        makeTaskContextStage2(task, secondPass, count)
+        makeTaskContextStage1(task, values)
+        makeTaskContextStage2(task)
 
         // verify that `when` guard, when specified, is satisfied
         if( !checkWhenGuard(task) )
@@ -515,6 +511,9 @@ class TaskProcessor {
             task.resolve(block)
         }
         else {
+            // -- prepend task config to arguments (for process function)
+            values.push(task.config)
+
             // -- resolve the task command script
             task.resolve(taskBody, values.toArray())
         }
@@ -630,17 +629,8 @@ class TaskProcessor {
         task.config.executor = task.processor.executor.name
 
         /*
-         * initialize the inputs/outputs for this task instance
+         * initialize the outputs for this task instance
          */
-        config.getInputs().each { InParam param ->
-            if( param instanceof TupleInParam )
-                param.inner.each { task.setInput(it)  }
-            else if( param instanceof EachInParam )
-                task.setInput(param.inner)
-            else
-                task.setInput(param)
-        }
-
         config.getOutputs().each { OutParam param ->
             if( param instanceof TupleOutParam ) {
                 param.inner.each { task.setOutput(it) }
@@ -1899,88 +1889,81 @@ class TaskProcessor {
         return script.join('\n')
     }
 
-    final protected int makeTaskContextStage1( TaskRun task, Map secondPass, List values ) {
-
-        final contextMap = task.context
-        int count = 0
-
-        task.inputs.keySet().each { InParam param ->
-
-            // add the value to the task instance
-            def bindObject = param.getBindObject()
+    final protected void makeTaskContextStage1( TaskRun task, List values ) {
 
-            def val
-            if( bindObject instanceof Closure ) {
-                final cl = (Closure)bindObject.clone()
-                cl.delegate = task.context
-                cl.resolveStrategy = Closure.DELEGATE_FIRST
-                val = cl.call()
-            }
+        // collect params from process inputs
+        final params = []
+        for( InParam param : config.getInputs() ) {
+            if( param instanceof TupleInParam )
+                param.inner.each { params.add(it) }
+            else if( param instanceof EachInParam )
+                params.add(param.inner)
             else
-                val = bindObject
+                params.add(param)
+        }
 
-            log.trace "Process $name > binding param ${param.class.name} to ${val}"
+        // apply params to task config and context
+        for ( InParam param : params ) {
+
+            def val = param.decodeInputs(values)
 
             switch(param) {
                 case ValueInParam:
-                    contextMap.put( param.name, val )
+                    task.context.put( param.name, val )
                     break
 
                 case FileInParam:
-                    secondPass[param] = val
-                    return // <-- leave it, because we do not want to add this 'val' at this stage
+                    final allFiles = (List)task.config.get('files')
+                    allFiles.add( new TaskFileInput(val, param.isPathQualifier(), param.getName(), param.getOptions()) )
+                    break
 
                 case StdInParam:
+                    task.config.put('stdin', val)
+                    break
+
                 case EnvInParam:
-                    // nothing to do
+                    final allEnvs = task.config.get('env')
+                    allEnvs.put( param.name, val )
                     break
 
                 default:
                     throw new IllegalStateException("Unsupported input param type: ${param?.class?.simpleName}")
             }
-
-            // add the value to the task instance context
-            task.setInput(param, val)
         }
-
-        return count
     }
 
-    final protected void makeTaskContextStage2( TaskRun task, Map secondPass, int count ) {
+    final protected void makeTaskContextStage2( TaskRun task ) {
 
         final ctx = task.context
+        final allFileInputs = (List)task.config.get('files')
         final allNames = new HashMap()
+        int count = 0
 
         final FilePorter.Batch batch = session.filePorter.newBatch(executor.getStageDir())
 
-        // -- all file parameters are processed in a second pass
-        //    so that we can use resolve the variables that eventually are in the file name
-        for( Map.Entry entry : secondPass.entrySet() ) {
-            final param = entry.getKey()
-            final val = entry.getValue()
-            final fileParam = param as FileInParam
-            final normalized = normalizeInputToFiles(val, count, fileParam.isPathQualifier(), batch)
-            final resolved = expandWildcards( fileParam.getFilePattern(ctx), normalized )
+        // -- resolve input files against the task context
+        for( def fileInput : allFileInputs ) {
+            final normalized = normalizeInputToFiles(fileInput.getValue(ctx), count, fileInput.isPathQualifier(), batch)
+            final resolved = expandWildcards( fileInput.getFilePattern(ctx), normalized )
 
-            if( !param.isValidArity(resolved.size()) )
+            if( !fileInput.isValidArity(resolved.size()) )
                 throw new IllegalArityException("Incorrect number of input files for process `${safeTaskName(task)}` -- expected ${param.arity}, found ${resolved.size()}")
 
-            ctx.put( param.name, singleItemOrList(resolved, param.isSingle(), task.type) )
+            ctx.put( fileInput.name, singleItemOrList(resolved, fileInput.isSingle(), task.type) )
             count += resolved.size()
             for( FileHolder item : resolved ) {
                 Integer num = allNames.getOrCreate(item.stageName, 0) +1
                 allNames.put(item.stageName,num)
             }
 
-            // add the value to the task instance context
-            task.setInput(param, resolved)
+            task.inputFiles.addAll(resolved)
         }
 
         // -- set the delegate map as context in the task config
         //    so that lazy directives will be resolved against it
         task.config.context = ctx
 
-        // check conflicting file names
+        // -- check conflicting file names
         def conflicts = allNames.findAll { name, num -> num>1 }
         if( conflicts ) {
             log.debug("Process $name > collision check staging file names: $allNames")
@@ -2011,10 +1994,8 @@ class TaskProcessor {
             keys << task.getContainerFingerprint()
 
         // add all the input name-value pairs to the key generator
-        for( Map.Entry it : task.inputs ) {
-            keys.add( it.key.name )
-            keys.add( it.value )
-        }
+        for( FileHolder it : task.inputFiles )
+            keys.add( it )
 
         // add all variable references in the task script but not declared as input/output
         def vars = getTaskGlobalVars(task)
diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy
index 778df8b419..57f592edad 100644
--- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy
@@ -38,13 +38,9 @@ import nextflow.script.BodyDef
 import nextflow.script.ScriptType
 import nextflow.script.TaskClosure
 import nextflow.script.bundle.ResourcesBundle
-import nextflow.script.params.EnvInParam
 import nextflow.script.params.EnvOutParam
-import nextflow.script.params.FileInParam
 import nextflow.script.params.FileOutParam
-import nextflow.script.params.InParam
 import nextflow.script.params.OutParam
-import nextflow.script.params.StdInParam
 import nextflow.script.params.ValueOutParam
 import nextflow.spack.SpackCache
 import org.codehaus.groovy.runtime.MethodClosure
@@ -85,9 +81,9 @@ class TaskRun implements Cloneable {
     TaskProcessor processor
 
     /**
-     * Holds the input value(s) for each task input parameter
+     * The list of resolved input files
      */
-    Map inputs = [:]
+    List inputFiles = []
 
     /**
      * Holds the output value(s) for each task output parameter
@@ -95,18 +91,6 @@ class TaskRun implements Cloneable {
     Map outputs = [:]
 
 
-    void setInput( InParam param, Object value = null ) {
-        assert param
-
-        inputs[param] = value
-
-        // copy the value to the task 'input' attribute
-        // it will be used to pipe it to the process stdin
-        if( param instanceof StdInParam) {
-            stdin = value
-        }
-    }
-
     void setOutput( OutParam param, Object value = null ) {
         assert param
         outputs[param] = value
@@ -114,9 +98,11 @@ class TaskRun implements Cloneable {
 
 
     /**
-     * The value to be piped to the process stdin
+     * @return The value to be piped to the process stdin
      */
-    def stdin
+    def getStdin() {
+        config.get('stdin')
+    }
 
     /**
      * The exit code returned by executing the task script
@@ -415,33 +401,20 @@ class TaskRun implements Cloneable {
         return false
     }
 
-    Map> getInputFiles() {
-        (Map>) getInputsByType( FileInParam )
-    }
-
     /**
      * Return the list of all input files staged as inputs by this task execution
      */
     List getStagedInputs()  {
-        getInputFiles()
-                .values()
-                .flatten()
-                .collect { it.stageName }
+        inputFiles.collect { it.stageName }
     }
 
     /**
      * @return A map object containing all the task input files as  pairs
      */
     Map getInputFilesMap() {
-
         def result = [:]
-        def allFiles = getInputFiles().values()
-        for( List entry : allFiles ) {
-            if( entry ) for( FileHolder it : entry ) {
-                result[ it.stageName ] = it.storePath
-            }
-        }
-
+        for( FileHolder it : inputFiles )
+            result[ it.stageName ] = it.storePath
         return result
     }
 
@@ -465,25 +438,9 @@ class TaskRun implements Cloneable {
     }
 
     /**
-     * Get the map of *input* objects by the given {@code InParam} type
+     * Get the map of *output* objects by the given {@code OutParam} type
      *
-     * @param types One or more subclass of {@code InParam}
-     * @return An associative array containing all the objects for the specified type
-     */
-    def  Map getInputsByType( Class... types ) {
-
-        def result = [:]
-        for( def it : inputs ) {
-            if( types.contains(it.key.class) )
-                result << it
-        }
-        return result
-    }
-
-    /**
-     * Get the map of *output* objects by the given {@code InParam} type
-     *
-     * @param types One or more subclass of {@code InParam}
+     * @param types One or more subclass of {@code OutParam}
      * @return An associative array containing all the objects for the specified type
      */
     def  Map getOutputsByType( Class... types ) {
@@ -500,9 +457,9 @@ class TaskRun implements Cloneable {
      */
     protected Map getInputEnvironment() {
         final Map environment = [:]
-        getInputsByType( EnvInParam ).each { param, value ->
-            environment.put( param.name, value?.toString() )
-        }
+        final allEnvs = config.get('env')
+        for( def key : allEnvs.keySet() )
+            environment.put(key, allEnvs.get(key))
         return environment
     }
 
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessConfig.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessConfig.groovy
index 40e21830a6..7153a7c513 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessConfig.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessConfig.groovy
@@ -134,11 +134,18 @@ class ProcessConfig implements Map, Cloneable {
             case 'cacheable':
                 return isCacheable()
 
+            case 'env':
             case 'ext':
-                if( !configProperties.containsKey('ext') ) {
-                    configProperties.put('ext', new HashMap())
+                if( !configProperties.containsKey(name) ) {
+                    configProperties.put(name, new HashMap())
                 }
-                return configProperties.get('ext')
+                return configProperties.get(name)
+
+            case 'files':
+                if( !configProperties.containsKey(name) ) {
+                    configProperties.put(name, new ArrayList())
+                }
+                return configProperties.get(name)
 
             default:
                 if( configProperties.containsKey(name) )
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessInputsBuilder.groovy b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessInputsBuilder.groovy
index f61bf1483d..72e3d9a5d5 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessInputsBuilder.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessInputsBuilder.groovy
@@ -17,7 +17,7 @@
 package nextflow.script.dsl
 
 import groovy.util.logging.Slf4j
-import nextflow.script.params.*
+import nextflow.processor.TaskFileInput
 import nextflow.script.ProcessConfig
 
 /**
@@ -34,26 +34,23 @@ class ProcessInputsBuilder {
         this.config = config
     }
 
-    void env( Object obj ) {
-        new EnvInParam(config).bind(obj)
+    void env( String name, Object obj ) {
+        final allEnvs = (Map)config.env
+        allEnvs.put(name, obj)
     }
 
     void file( Object obj ) {
-        new FileInParam(config).bind(obj)
+        final allFiles = (List)config.files
+        allFiles.add(new TaskFileInput(obj, false, [:]))
     }
 
-    void path( Map opts=null, Object obj ) {
-        new FileInParam(config)
-                .setPathQualifier(true)
-                .setOptions(opts)
-                .bind(obj)
+    void path( Map opts=[:], Object obj ) {
+        final allFiles = (List)config.files
+        allFiles.add(new TaskFileInput(obj, true, opts))
     }
 
-    void stdin( Object obj = null ) {
-        def result = new StdInParam(config)
-        if( obj )
-            result.bind(obj)
-        result
+    void stdin( Object obj ) {
+        config.put('stdin', obj)
     }
 
     ProcessConfig getConfig() {
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/FileInParam.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/FileInParam.groovy
index 397a759b15..ce9c984d7c 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/params/FileInParam.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/params/FileInParam.groovy
@@ -18,8 +18,6 @@ package nextflow.script.params
 
 import groovy.transform.InheritConstructors
 import groovy.util.logging.Slf4j
-import nextflow.NF
-import nextflow.script.TokenVar
 
 /**
  * Represents a process *file* input parameter
@@ -28,36 +26,16 @@ import nextflow.script.TokenVar
  */
 @Slf4j
 @InheritConstructors
-class FileInParam extends BaseInParam implements ArityParam, PathQualifier {
-
-    protected filePattern
+class FileInParam extends BaseInParam implements PathQualifier {
 
     private boolean pathQualifier
 
+    private Map options
+
     @Override String getTypeName() { pathQualifier ? 'path' : 'file' }
 
     @Override String getTypeSimpleName() { getTypeName() + "inparam" }
 
-    /**
-     * Define the file name
-     */
-    FileInParam name( obj ) {
-        if( pathQualifier )
-            throw new MissingMethodException("name", this.class, [String] as Object[])
-
-        if( obj instanceof String ) {
-            filePattern = obj
-            return this
-        }
-
-        if( obj instanceof GString ) {
-            filePattern = obj
-            return this
-        }
-
-        throw new IllegalArgumentException()
-    }
-
     String getName() {
         if( bindObject instanceof Map ) {
             assert !pathQualifier
@@ -80,39 +58,6 @@ class FileInParam extends BaseInParam implements ArityParam, PathQualifier {
         return this
     }
 
-    String getFilePattern(Map ctx = null) {
-
-        if( filePattern != null  )
-            return resolve(ctx,filePattern)
-
-        if( bindObject instanceof Map ) {
-            assert !pathQualifier
-            def entry = bindObject.entrySet().first()
-            return resolve(ctx, entry?.value)
-        }
-
-        if( bindObject instanceof TokenVar )
-            return filePattern = '*'
-
-        if( bindObject != null )
-            return resolve(ctx, bindObject)
-
-        return filePattern = '*'
-    }
-
-    private resolve( Map ctx, value ) {
-        if( value instanceof GString ) {
-            value.cloneAsLazy(ctx)
-        }
-
-        else if( value instanceof Closure ) {
-            return ctx.with(value)
-        }
-
-        else
-            return value
-    }
-
     @Override
     FileInParam setPathQualifier(boolean flag) {
         pathQualifier = flag
@@ -124,26 +69,10 @@ class FileInParam extends BaseInParam implements ArityParam, PathQualifier {
 
     @Override
     FileInParam setOptions(Map opts) {
-        (FileInParam)super.setOptions(opts)
-    }
-
-    /**
-     * Defines the `stageAs:` option to define the input file stage name pattern
-     *
-     * @param value
-     *      A string representing the target file name or a file name pattern
-     *      ie. containing the star `*` or question mark wildcards
-     * @return
-     *      The param instance itself
-     */
-    FileInParam setStageAs(String value) {
-        this.filePattern = value
+        this.options = opts
         return this
     }
 
-    FileInParam setName(String value) {
-        this.filePattern = value
-        return this
-    }
+    Map getOptions() { options }
 
 }
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/FileOutParam.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/FileOutParam.groovy
index 97721ea7b2..2a3f94bb6f 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/params/FileOutParam.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/params/FileOutParam.groovy
@@ -24,6 +24,7 @@ import groovy.util.logging.Slf4j
 import nextflow.NF
 import nextflow.exception.IllegalFileException
 import nextflow.file.FilePatternSplitter
+import nextflow.processor.PathArityAware
 import nextflow.script.TokenVar
 import nextflow.util.BlankSeparatedList
 /**
@@ -33,7 +34,7 @@ import nextflow.util.BlankSeparatedList
  */
 @Slf4j
 @InheritConstructors
-class FileOutParam extends BaseOutParam implements OutParam, ArityParam, OptionalParam, PathQualifier {
+class FileOutParam extends BaseOutParam implements OutParam, OptionalParam, PathArityAware, PathQualifier {
 
     /**
      * ONLY FOR TESTING DO NOT USE

From cf0e4b2ce1fc8e757cb52df11a7ffa26b0ab12f7 Mon Sep 17 00:00:00 2001
From: Ben Sherman 
Date: Sat, 2 Dec 2023 16:11:52 -0600
Subject: [PATCH 11/36] Fix process input channel logic

Signed-off-by: Ben Sherman 
---
 docs/operator.md                              | 12 ++--
 .../src/main/groovy/nextflow/dag/DAG.groovy   |  4 +-
 .../nextflow/extension/CombineOp.groovy       | 20 ++++--
 .../nextflow/extension/DataflowHelper.groovy  |  4 +-
 .../extension/DefaultMergeClosure.groovy      | 26 +++----
 .../groovy/nextflow/extension/MergeOp.groovy  | 12 ++--
 .../nextflow/extension/OperatorImpl.groovy    | 21 +++---
 .../processor/InvokeTaskAdapter.groovy        | 65 -----------------
 .../nextflow/processor/TaskProcessor.groovy   | 58 ++++++++-------
 .../groovy/nextflow/script/ProcessDef.groovy  | 72 +++++++++++--------
 .../nextflow/script/params/EachInParam.groovy | 22 ++----
 11 files changed, 128 insertions(+), 188 deletions(-)
 delete mode 100644 modules/nextflow/src/main/groovy/nextflow/processor/InvokeTaskAdapter.groovy

diff --git a/docs/operator.md b/docs/operator.md
index 9593fcc248..028099ecec 100644
--- a/docs/operator.md
+++ b/docs/operator.md
@@ -334,6 +334,11 @@ A second version of the `combine` operator allows you to combine items that shar
 :language: console
 ```
 
+:::{versionadded} 24.10.0
+:::
+
+By default, the `combine` operator flattens list items into the resulting tuple. You can set `flat: false` to preserve nested list items.
+
 See also [join](#join) and [cross](#cross).
 
 (operator-concat)=
@@ -847,13 +852,6 @@ An optional closure can be provided to customise the items emitted by the result
 :language: console
 ```
 
-Available options:
-
-`flat`
-: :::{versionadded} 24.10.0
-  :::
-: When `true`, automatically flattens merged items by one level (default: `true`). This option is ignored when a mapping closure is specified.
-
 :::{danger}
 In general, the use of the `merge` operator is discouraged. Processes and channel operators are not guaranteed to emit items in the order that they were received, as they are executed concurrently. Therefore, if you try to merge output channels from different processes, the resulting channel may be different on each run, which will cause resumed runs to {ref}`not work properly `.
 
diff --git a/modules/nextflow/src/main/groovy/nextflow/dag/DAG.groovy b/modules/nextflow/src/main/groovy/nextflow/dag/DAG.groovy
index 323018e71c..bc92a3b3ce 100644
--- a/modules/nextflow/src/main/groovy/nextflow/dag/DAG.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/dag/DAG.groovy
@@ -97,7 +97,7 @@ class DAG {
      */
     void addProcessNode( String label, InputsList inputs, OutputsList outputs, TaskProcessor process=null ) {
         assert label
-        assert inputs != null
+        assert inputs
         assert outputs
         addVertex( Type.PROCESS, label, normalizeInputs(inputs), normalizeOutputs(outputs), process )
     }
@@ -111,7 +111,7 @@ class DAG {
      */
     void addOperatorNode( String label, inputs, outputs, List operators=null )  {
         assert label
-        assert inputs != null
+        assert inputs
         addVertex(Type.OPERATOR, label, normalizeChannels(inputs), normalizeChannels(outputs), operators )
     }
 
diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/CombineOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/CombineOp.groovy
index 729e500004..c06b2ebd56 100644
--- a/modules/nextflow/src/main/groovy/nextflow/extension/CombineOp.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/extension/CombineOp.groovy
@@ -54,7 +54,9 @@ class CombineOp {
 
     private List pivot = NONE
 
-    CombineOp(DataflowReadChannel left, Object right) {
+    private boolean flat = true
+
+    CombineOp(DataflowReadChannel left, Object right, Map opts) {
 
         leftChannel = left
 
@@ -72,6 +74,11 @@ class CombineOp {
                 throw new IllegalArgumentException("Not a valid argument for 'combine' operator [${right?.class?.simpleName}]: ${right} -- Use a List or a channel instead. ")
         }
 
+        if( opts?.by != null )
+            pivot = opts.by as List
+
+        if( opts?.flat != null )
+            flat = opts.flat
     }
 
     CombineOp setPivot( pivot ) {
@@ -103,7 +110,8 @@ class CombineOp {
         opts.onComplete = {
             if( stopCount.decrementAndGet()==0) {
                 target << Channel.STOP
-            }}
+            }
+        }
 
         return opts
     }
@@ -113,8 +121,8 @@ class CombineOp {
     def tuple( List p, a, b ) {
         List result = new LinkedList()
         result.addAll(p)
-        addToList(result, a)
-        addToList(result, b)
+        addToList(result, a, flat)
+        addToList(result, b, flat)
 
         result.size()==1 ? result[0] : result
     }
@@ -143,7 +151,7 @@ class CombineOp {
             return
         }
 
-        throw new IllegalArgumentException("Not a valid spread operator index: $index")
+        throw new IllegalArgumentException("Not a valid combine operator index: $index")
     }
 
     DataflowWriteChannel apply() {
@@ -162,7 +170,7 @@ class CombineOp {
         }
 
         else
-            throw new IllegalArgumentException("Not a valid spread operator state -- Missing right operand")
+            throw new IllegalArgumentException("Not a valid combine operator state -- Missing right operand")
 
         return target
     }
diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy
index dd4cdd5b5a..bb10a8c868 100644
--- a/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy
@@ -381,8 +381,8 @@ class DataflowHelper {
 
     @PackageScope
     @CompileStatic
-    static void addToList(List result, entry)  {
-        if( entry instanceof List ) {
+    static void addToList(List result, Object entry, boolean flat=true)  {
+        if( flat && entry instanceof List ) {
             result.addAll(entry)
         }
         else {
diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/DefaultMergeClosure.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/DefaultMergeClosure.groovy
index 99062f3518..e904834b89 100644
--- a/modules/nextflow/src/main/groovy/nextflow/extension/DefaultMergeClosure.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/extension/DefaultMergeClosure.groovy
@@ -25,12 +25,9 @@ class DefaultMergeClosure extends Closure {
 
     private int numOfParams
 
-    private boolean flat
-
-    DefaultMergeClosure(int n, boolean flat) {
-        super(null, null)
-        this.numOfParams = n
-        this.flat = flat
+    DefaultMergeClosure(int n) {
+        super(null, null);
+        numOfParams = n
     }
 
     @Override
@@ -45,12 +42,12 @@ class DefaultMergeClosure extends Closure {
 
     @Override
     public void setDelegate(final Object delegate) {
-        super.setDelegate(delegate)
+        super.setDelegate(delegate);
     }
 
     @Override
     public void setResolveStrategy(final int resolveStrategy) {
-        super.setResolveStrategy(resolveStrategy)
+        super.setResolveStrategy(resolveStrategy);
     }
 
     @Override
@@ -61,15 +58,10 @@ class DefaultMergeClosure extends Closure {
     @Override
     public Object call(final Object... args) {
         final result = []
-        for( int i=0; i others
-    private boolean flat
     private Closure closure
 
-    MergeOp(DataflowReadChannel source, List others, Map opts=null, Closure closure=null) {
+    MergeOp(final DataflowReadChannel source, final List others, final Closure closure=null) {
         this.source = source
         this.others = others
-        this.flat = opts?.flat!=null ? opts?.flat : true
         this.closure = closure
     }
 
-    MergeOp(DataflowReadChannel source, DataflowReadChannel other, Map opts=null, Closure closure=null) {
-        this(source, Collections.singletonList(other), opts, closure)
+    MergeOp(final DataflowReadChannel source, final DataflowReadChannel other, final Closure closure=null ) {
+        this.source = source
+        this.others = Collections.singletonList(other)
+        this.closure = closure
     }
 
     DataflowWriteChannel apply() {
         final result = CH.createBy(source)
         final List inputs = new ArrayList(1 + others.size())
-        final action = closure ? new ChainWithClosure<>(closure) : new DefaultMergeClosure(1 + others.size(), flat)
+        final action = closure ? new ChainWithClosure<>(closure) : new DefaultMergeClosure(1 + others.size())
         inputs.add(source)
         inputs.addAll(others)
         final listener = stopErrorListener(source,result)
diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy
index 2e2924bf6f..e622173307 100644
--- a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy
@@ -715,9 +715,8 @@ class OperatorImpl {
     DataflowWriteChannel combine( DataflowReadChannel left, Map params, Object right ) {
         checkParams('combine', params, [flat:Boolean, by: [List,Integer]])
 
-        final op = new CombineOp(left,right)
+        final op = new CombineOp(left,right,params)
         OpCall.current.get().inputs.addAll(op.inputs)
-        if( params?.by != null ) op.pivot = params.by
         final target = op.apply()
         return target
     }
@@ -1077,18 +1076,24 @@ class OperatorImpl {
     }
 
     // NO DAG
-    DataflowWriteChannel merge(DataflowReadChannel source, Map opts=null, DataflowReadChannel other, Closure closure=null) {
-        new MergeOp(source, other, opts, closure).apply()
+    DataflowWriteChannel merge(final DataflowReadChannel source, final DataflowReadChannel other, final Closure closure=null) {
+        final result = CH.createBy(source)
+        final inputs = [source, other]
+        final action = closure ? new ChainWithClosure<>(closure) : new DefaultMergeClosure(inputs.size())
+        final listener = stopErrorListener(source,result)
+        final params = createOpParams(inputs, result, listener)
+        newOperator(params, action)
+        return result;
     }
 
     // NO DAG
-    DataflowWriteChannel merge(DataflowReadChannel source, Map opts=null, DataflowReadChannel... others) {
-        new MergeOp(source, others as List, opts).apply()
+    DataflowWriteChannel merge(final DataflowReadChannel source, final DataflowReadChannel... others) {
+        new MergeOp(source,others as List).apply()
     }
 
     // NO DAG
-    DataflowWriteChannel merge(DataflowReadChannel source, Map opts=null, List others, Closure closure=null) {
-        new MergeOp(source, others, opts, closure).apply()
+    DataflowWriteChannel merge(final DataflowReadChannel source, final List others, final Closure closure=null) {
+        new MergeOp(source,others,closure).apply()
     }
 
     DataflowWriteChannel randomSample(DataflowReadChannel source, int n, Long seed = null) {
diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/InvokeTaskAdapter.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/InvokeTaskAdapter.groovy
deleted file mode 100644
index 3c26ca54f6..0000000000
--- a/modules/nextflow/src/main/groovy/nextflow/processor/InvokeTaskAdapter.groovy
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Copyright 2013-2023, Seqera Labs
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package nextflow.processor
-
-import groovy.transform.CompileStatic
-
-/**
- * Adapter closure to call the {@link TaskProcessor#invokeTask(java.lang.Object)} method
- */
-@CompileStatic
-class InvokeTaskAdapter extends Closure {
-
-    private int numOfParams
-
-    private TaskProcessor processor
-
-    InvokeTaskAdapter(TaskProcessor p, int n) {
-        super(null, null);
-        processor = p
-        numOfParams = n
-    }
-
-    @Override
-    int getMaximumNumberOfParameters() {
-        numOfParams
-    }
-
-    @Override
-    Class[] getParameterTypes() {
-        def result = new Class[numOfParams]
-        for( int i=0; i $name with params=$params; values=$values"
 
@@ -491,10 +483,6 @@ class TaskProcessor {
         // -- set the task instance as the current in this thread
         currentTask.set(task)
 
-        // -- add arguments to task context (for process function)
-        for( int i = 1; i < config.params.length; i++ )
-            task.context.put(config.params[i], values[i-1])
-
         // -- validate input lengths
         validateInputTuples(values)
 
@@ -1891,6 +1879,12 @@ class TaskProcessor {
 
     final protected void makeTaskContextStage1( TaskRun task, List values ) {
 
+        final allValues = [:]
+
+        // add arguments to task config (for process function)
+        for( int i = 1; config.params!=null && i < config.params.length; i++ )
+            allValues.put(config.params[i], values[i-1])
+
         // collect params from process inputs
         final params = []
         for( InParam param : config.getInputs() ) {
@@ -1902,14 +1896,14 @@ class TaskProcessor {
                 params.add(param)
         }
 
-        // apply params to task config and context
+        // add param values to task config and context
         for ( InParam param : params ) {
 
             def val = param.decodeInputs(values)
 
             switch(param) {
                 case ValueInParam:
-                    task.context.put( param.name, val )
+                    allValues.put( param.name, val )
                     break
 
                 case FileInParam:
@@ -1930,6 +1924,9 @@ class TaskProcessor {
                     throw new IllegalStateException("Unsupported input param type: ${param?.class?.simpleName}")
             }
         }
+
+        task.config.put('vals', allValues)
+        task.context.putAll(allValues)
     }
 
     final protected void makeTaskContextStage2( TaskRun task ) {
@@ -1993,9 +1990,11 @@ class TaskProcessor {
         if( task.isContainerEnabled() )
             keys << task.getContainerFingerprint()
 
-        // add all the input name-value pairs to the key generator
-        for( FileHolder it : task.inputFiles )
-            keys.add( it )
+        // add task inputs
+        keys.add( task.config.get('vals') )
+        keys.add( task.inputFiles )
+        keys.add( task.getInputEnvironment() )
+        keys.add( task.stdin )
 
         // add all variable references in the task script but not declared as input/output
         def vars = getTaskGlobalVars(task)
@@ -2275,10 +2274,9 @@ class TaskProcessor {
             // task index must be created here to guarantee consistent ordering
             // with the sequence of messages arrival since this method is executed in a thread safe manner
             final params = new TaskStartParams(TaskId.next(), indexCount.incrementAndGet())
-            final args = source != null ? messages.first() : []
             final result = new ArrayList(2)
             result[0] = params
-            result[1] = args
+            result[1] = messages.first()
             return result
         }
 
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy
index 56390cb9a1..5343181837 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy
@@ -153,14 +153,7 @@ class ProcessDef extends BindableDef implements IterableDef, ChainableDef {
         initialize()
 
         // create merged input channel
-        final source = getSourceChannel(args)
-
-        // set input channels
-        for( int i=0; i getInChannel(ch))
-        if( inputs.findAll(ch -> CH.isChannelQueue(ch)).size() > 1 )
-            throw new ScriptRuntimeException("Process `$name` received multiple queue channel inputs which is not allowed")
-
-        // merge input channels
-        // TODO: skip `each` inputs
-        def source = CH.getReadChannel(new MergeOp(inputs.first(), inputs[1.. each param '${param.name}' at index ${i} -- ${param.dump()}"
-            source = CH.getReadChannel(new CombineOp(source, param.getInChannel()))
+    private DataflowReadChannel collectInputs(Object[] args0) {
+        final args = ChannelOut.spread(args0)
+        final hasDeclaredInputs = config.params==null
+        if( hasDeclaredInputs && args.size() != declaredInputs.size() )
+            throw new ScriptRuntimeException(missMatchErrMessage(processName, declaredInputs.size(), args.size()))
+
+        // emit value channel if process has no inputs
+        if( args.size() == 0 ) {
+            final source = CH.value()
+            source.bind([])
+            return source
         }
 
-        return source
+        // set input channels
+        for( int i = 0; i < declaredInputs.size(); i++ ) {
+            final param = (declaredInputs[i] as BaseInParam)
+            param.setFrom(args[i])
+            param.init()
+        }
+
+        // normalize args into channels
+        final inputs = hasDeclaredInputs
+            ? declaredInputs.getChannels()
+            : args.collect(ch -> getInChannel(ch))
+
+        // make sure no more than one queue channel is provided
+        int count = 0
+        for( int i = 0; i < inputs.size(); i++ )
+            if( CH.isChannelQueue(inputs[i]) && (!hasDeclaredInputs || declaredInputs[i] !instanceof EachInParam) )
+                count += 1
+
+        if( count > 1 )
+            throw new ScriptRuntimeException("Process `$name` received multiple queue channel inputs which is not allowed -- consider combining these channels explicitly using the `combine` or `join` operator")
+
+        // combine input channels
+        def result = inputs.first()
+
+        if( inputs.size() == 1 )
+            return result.chainWith { it instanceof Collection ? it : [it] }
+
+        for( int i = 1; i < inputs.size(); i++ )
+            result = CH.getReadChannel(new CombineOp(result, inputs[i], [flat: false]).apply())
+
+        return result
     }
 
     private DataflowReadChannel getInChannel(Object obj) {
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/EachInParam.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/EachInParam.groovy
index 2404fca43d..1ef40a1f6c 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/params/EachInParam.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/params/EachInParam.groovy
@@ -20,10 +20,7 @@ import groovy.transform.InheritConstructors
 import groovy.transform.PackageScope
 import groovy.util.logging.Slf4j
 import groovyx.gpars.dataflow.DataflowReadChannel
-import groovyx.gpars.dataflow.DataflowVariable
-import groovyx.gpars.dataflow.expression.DataflowExpression
 import nextflow.extension.CH
-import nextflow.extension.ToListOp
 import nextflow.script.TokenFileCall
 import nextflow.script.TokenPathCall
 
@@ -84,20 +81,13 @@ class EachInParam extends BaseInParam {
 
     @PackageScope
     DataflowReadChannel normalizeToVariable( value ) {
-        def result
-        if( value instanceof DataflowExpression ) {
-            result = value
+        if( value instanceof Collection ) {
+            final result = CH.create()
+            CH.emitAndClose(result, value as List)
+            return CH.getReadChannel(result)
         }
-        else if( CH.isChannel(value) ) {
-            def read = CH.getReadChannel(value)
-            result = new ToListOp(read).apply()
-        }
-        else {
-            result = new DataflowVariable()
-            result.bind(value)
-        }
-
-        return result.chainWith { it instanceof Collection || it == null ? it : [it] }
+        else
+            return CH.getReadChannel(value)
     }
 
 }

From 0c490e8595410bb646bc94d315f26cd0d1798bf3 Mon Sep 17 00:00:00 2001
From: Ben Sherman 
Date: Wed, 6 Dec 2023 15:42:33 -0600
Subject: [PATCH 12/36] Fix bugs

Signed-off-by: Ben Sherman 
---
 .../src/main/groovy/nextflow/dag/DAG.groovy       |  4 ++--
 .../nextflow/processor/TaskFileInput.groovy       | 13 ++++++++-----
 .../nextflow/processor/TaskProcessor.groovy       | 12 ++++++------
 .../groovy/nextflow/script/ProcessConfig.groovy   | 15 +++++++--------
 .../main/groovy/nextflow/script/ProcessDef.groovy |  4 ++--
 .../nextflow/script/dsl/ProcessBuilder.groovy     |  4 ++--
 .../script/dsl/ProcessInputsBuilder.groovy        | 12 +++++++-----
 7 files changed, 34 insertions(+), 30 deletions(-)

diff --git a/modules/nextflow/src/main/groovy/nextflow/dag/DAG.groovy b/modules/nextflow/src/main/groovy/nextflow/dag/DAG.groovy
index bc92a3b3ce..5447d3b5e7 100644
--- a/modules/nextflow/src/main/groovy/nextflow/dag/DAG.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/dag/DAG.groovy
@@ -97,7 +97,7 @@ class DAG {
      */
     void addProcessNode( String label, InputsList inputs, OutputsList outputs, TaskProcessor process=null ) {
         assert label
-        assert inputs
+        assert inputs!=null
         assert outputs
         addVertex( Type.PROCESS, label, normalizeInputs(inputs), normalizeOutputs(outputs), process )
     }
@@ -111,7 +111,7 @@ class DAG {
      */
     void addOperatorNode( String label, inputs, outputs, List operators=null )  {
         assert label
-        assert inputs
+        assert inputs!=null
         addVertex(Type.OPERATOR, label, normalizeChannels(inputs), normalizeChannels(outputs), operators )
     }
 
diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskFileInput.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskFileInput.groovy
index b93064ae0e..a1e8909cce 100644
--- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskFileInput.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskFileInput.groovy
@@ -49,17 +49,20 @@ class TaskFileInput implements PathArityAware {
         return coerceToPath
     }
 
-    String getName() {
-        return name
+    String getName(Map ctx) {
+        if( name != null )
+            return name
+
+        if( value != null )
+            return resolve(ctx, value)
+
+        return null
     }
 
     String getFilePattern(Map ctx) {
         if( filePattern != null )
             return resolve(ctx, filePattern)
 
-        if( value != null )
-            return resolve(ctx, value)
-
         return filePattern = '*'
     }
 
diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy
index a48ae8114d..29ded3970b 100644
--- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy
@@ -522,9 +522,9 @@ class TaskProcessor {
 
     protected void validateInputTuples( List values ) {
 
-        def declaredSets = getDeclaredInputTuple()
-        for( int i=0; i, Cloneable {
     /**
      * List of parameter names defined by a process function.
      */
-    String[] params
+    private String[] params
 
     /**
      * List of process input definitions
@@ -118,6 +118,12 @@ class ProcessConfig implements Map, Cloneable {
         return this
     }
 
+    @PackageScope
+    ProcessConfig setParams(String[] params) {
+        this.params = params
+        return this
+    }
+
     @Override
     Object getProperty( String name ) {
 
@@ -134,19 +140,12 @@ class ProcessConfig implements Map, Cloneable {
             case 'cacheable':
                 return isCacheable()
 
-            case 'env':
             case 'ext':
                 if( !configProperties.containsKey(name) ) {
                     configProperties.put(name, new HashMap())
                 }
                 return configProperties.get(name)
 
-            case 'files':
-                if( !configProperties.containsKey(name) ) {
-                    configProperties.put(name, new ArrayList())
-                }
-                return configProperties.get(name)
-
             default:
                 if( configProperties.containsKey(name) )
                     return configProperties.get(name)
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy
index 5343181837..22c47da095 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy
@@ -195,7 +195,7 @@ class ProcessDef extends BindableDef implements IterableDef, ChainableDef {
 
     private DataflowReadChannel collectInputs(Object[] args0) {
         final args = ChannelOut.spread(args0)
-        final hasDeclaredInputs = config.params==null
+        final hasDeclaredInputs = config.getParams()==null
         if( hasDeclaredInputs && args.size() != declaredInputs.size() )
             throw new ScriptRuntimeException(missMatchErrMessage(processName, declaredInputs.size(), args.size()))
 
@@ -231,7 +231,7 @@ class ProcessDef extends BindableDef implements IterableDef, ChainableDef {
         def result = inputs.first()
 
         if( inputs.size() == 1 )
-            return result.chainWith { it instanceof Collection ? it : [it] }
+            return result.chainWith( it -> [it] )
 
         for( int i = 1; i < inputs.size(); i++ )
             result = CH.getReadChannel(new CombineOp(result, inputs[i], [flat: false]).apply())
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessBuilder.groovy b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessBuilder.groovy
index c4721f5653..710be1eef7 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessBuilder.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessBuilder.groovy
@@ -428,7 +428,7 @@ class ProcessBuilder {
         new FileInParam(config).bind(obj)
     }
 
-    InParam _in_path( Map opts=null, Object obj ) {
+    InParam _in_path( Map opts=[:], Object obj ) {
         new FileInParam(config)
                 .setPathQualifier(true)
                 .setOptions(opts)
@@ -531,7 +531,7 @@ class ProcessBuilder {
     /// SCRIPT
 
     ProcessBuilder withParams(String[] params) {
-        config.params = params
+        config.setParams(params)
         return this
     }
 
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessInputsBuilder.groovy b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessInputsBuilder.groovy
index 72e3d9a5d5..22f05a0c43 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessInputsBuilder.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessInputsBuilder.groovy
@@ -16,6 +16,7 @@
 
 package nextflow.script.dsl
 
+import groovy.transform.CompileStatic
 import groovy.util.logging.Slf4j
 import nextflow.processor.TaskFileInput
 import nextflow.script.ProcessConfig
@@ -26,6 +27,7 @@ import nextflow.script.ProcessConfig
  * @author Ben Sherman 
  */
 @Slf4j
+@CompileStatic
 class ProcessInputsBuilder {
 
     private ProcessConfig config
@@ -35,18 +37,18 @@ class ProcessInputsBuilder {
     }
 
     void env( String name, Object obj ) {
-        final allEnvs = (Map)config.env
+        final allEnvs = (Map)config.get('env', [:])
         allEnvs.put(name, obj)
     }
 
     void file( Object obj ) {
-        final allFiles = (List)config.files
-        allFiles.add(new TaskFileInput(obj, false, [:]))
+        final allFiles = (List)config.get('files', [])
+        allFiles.add(new TaskFileInput(obj, false, null, [:]))
     }
 
     void path( Map opts=[:], Object obj ) {
-        final allFiles = (List)config.files
-        allFiles.add(new TaskFileInput(obj, true, opts))
+        final allFiles = (List)config.get('files', [])
+        allFiles.add(new TaskFileInput(obj, true, null, opts))
     }
 
     void stdin( Object obj ) {

From 8733ba6700fd96e0cf602d4914c10f17c709cf74 Mon Sep 17 00:00:00 2001
From: Ben Sherman 
Date: Thu, 7 Dec 2023 23:00:35 -0600
Subject: [PATCH 13/36] Refactor process inputs and outputs

Signed-off-by: Ben Sherman 
---
 .../groovy/nextflow/ast/ProcessFnXform.groovy |  36 +-
 .../src/main/groovy/nextflow/dag/DAG.groovy   |  38 +-
 .../groovy/nextflow/dag/NodeMarker.groovy     |   6 +-
 .../nextflow/processor/PublishDir.groovy      |   7 +-
 .../processor/TaskEnvCollector.groovy         |  58 ++
 .../processor/TaskFileCollector.groovy        | 146 +++++
 .../nextflow/processor/TaskProcessor.groovy   | 499 ++----------------
 .../groovy/nextflow/processor/TaskRun.groovy  |  57 +-
 .../groovy/nextflow/script/BaseScript.groovy  |  43 +-
 .../groovy/nextflow/script/ChannelOut.groovy  |  10 +-
 .../PathArityAware.groovy                     |  13 +-
 .../nextflow/script/ProcessConfig.groovy      |  37 +-
 .../groovy/nextflow/script/ProcessDef.groovy  |  86 +--
 .../nextflow/script/ProcessFactory.groovy     |   4 +-
 .../ProcessFileInput.groovy}                  |  42 +-
 .../nextflow/script/ProcessFileOutput.groovy  | 140 +++++
 .../nextflow/script/ProcessInput.groovy       |  78 +++
 .../nextflow/script/ProcessInputs.groovy      |  55 ++
 .../nextflow/script/ProcessOutput.groovy      | 172 ++++++
 .../nextflow/script/ProcessOutputs.groovy     |  59 +++
 .../nextflow/script/dsl/ProcessBuilder.groovy | 148 +-----
 .../nextflow/script/dsl/ProcessDsl.groovy     | 147 ++++++
 .../script/dsl/ProcessInputsBuilder.groovy    |  55 +-
 .../script/dsl/ProcessOutputsBuilder.groovy   |  76 +++
 .../script/params/DefaultOutParam.groovy      |  36 --
 .../script/params/FileOutParam.groovy         |  10 +-
 .../nextflow/script/params/InputsList.groovy  |  65 ---
 .../script/params/MissingParam.groovy         |  29 -
 .../nextflow/script/params/OutputsList.groovy |  56 --
 29 files changed, 1174 insertions(+), 1034 deletions(-)
 create mode 100644 modules/nextflow/src/main/groovy/nextflow/processor/TaskEnvCollector.groovy
 create mode 100644 modules/nextflow/src/main/groovy/nextflow/processor/TaskFileCollector.groovy
 rename modules/nextflow/src/main/groovy/nextflow/{processor => script}/PathArityAware.groovy (88%)
 rename modules/nextflow/src/main/groovy/nextflow/{processor/TaskFileInput.groovy => script/ProcessFileInput.groovy} (72%)
 create mode 100644 modules/nextflow/src/main/groovy/nextflow/script/ProcessFileOutput.groovy
 create mode 100644 modules/nextflow/src/main/groovy/nextflow/script/ProcessInput.groovy
 create mode 100644 modules/nextflow/src/main/groovy/nextflow/script/ProcessInputs.groovy
 create mode 100644 modules/nextflow/src/main/groovy/nextflow/script/ProcessOutput.groovy
 create mode 100644 modules/nextflow/src/main/groovy/nextflow/script/ProcessOutputs.groovy
 create mode 100644 modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessDsl.groovy
 create mode 100644 modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessOutputsBuilder.groovy
 delete mode 100644 modules/nextflow/src/main/groovy/nextflow/script/params/DefaultOutParam.groovy
 delete mode 100644 modules/nextflow/src/main/groovy/nextflow/script/params/InputsList.groovy
 delete mode 100644 modules/nextflow/src/main/groovy/nextflow/script/params/MissingParam.groovy
 delete mode 100644 modules/nextflow/src/main/groovy/nextflow/script/params/OutputsList.groovy

diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/ProcessFnXform.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/ProcessFnXform.groovy
index dcc8feee51..b872f815f2 100644
--- a/modules/nextflow/src/main/groovy/nextflow/ast/ProcessFnXform.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/ast/ProcessFnXform.groovy
@@ -83,28 +83,20 @@ class ProcessFnXform extends ClassCodeVisitorSupport {
                 fixDirectiveWithNegativeValue(stmt)
         }
 
-        // fix outputs
-        final outputs = annotation.getMember('outputs')
-        if( outputs != null && outputs instanceof ClosureExpression ) {
-            final block = (BlockStatement)outputs.getCode()
-            for( Statement stmt : block.getStatements() )
-                fixOutputMethod((ExpressionStatement)stmt)
-        }
-
-        // insert `task` method parameter
-        final params = method.getParameters() as List
-        params.push(new Parameter(new ClassNode(TaskConfig), 'task'))
-        method.setParameters(params as Parameter[])
-
         // TODO: append stub source
 
         // append method params
+        final params = method.getParameters() as List
         annotation.addMember( 'params', new ListExpression(
             params.collect(p -> (Expression)constX(p.getName()))
         ) )
 
         // append script source
         annotation.addMember( 'source', constX( getSource(method.getCode()) ) )
+
+        // prepend `task` method parameter
+        params.push(new Parameter(new ClassNode(TaskConfig), 'task'))
+        method.setParameters(params as Parameter[])
     }
 
     /**
@@ -138,24 +130,6 @@ class ProcessFnXform extends ClassCodeVisitorSupport {
         ) )
     }
 
-    private static final VALID_OUTPUT_METHODS = ['val','env','file','path','stdout','tuple']
-
-    /**
-     * Fix output method calls.
-     *
-     * @param stmt
-     */
-    protected void fixOutputMethod(ExpressionStatement stmt) {
-        final methodCall = (MethodCallExpression)stmt.getExpression()
-        final name = methodCall.getMethodAsString()
-        final args = (ArgumentListExpression)methodCall.getArguments()
-
-        if( name !in VALID_OUTPUT_METHODS )
-            syntaxError(stmt, "Invalid output method '${name}'")
-
-        methodCall.setMethod( constX('_out_' + name) )
-    }
-
     private String getSource(ASTNode node) {
         final buffer = new StringBuilder()
         final colx = node.getColumnNumber()
diff --git a/modules/nextflow/src/main/groovy/nextflow/dag/DAG.groovy b/modules/nextflow/src/main/groovy/nextflow/dag/DAG.groovy
index 5447d3b5e7..807af95545 100644
--- a/modules/nextflow/src/main/groovy/nextflow/dag/DAG.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/dag/DAG.groovy
@@ -30,14 +30,8 @@ import nextflow.NF
 import nextflow.extension.CH
 import nextflow.extension.DataflowHelper
 import nextflow.processor.TaskProcessor
-import nextflow.script.params.DefaultOutParam
-import nextflow.script.params.EachInParam
-import nextflow.script.params.InParam
-import nextflow.script.params.InputsList
-import nextflow.script.params.OutParam
-import nextflow.script.params.OutputsList
-import nextflow.script.params.TupleInParam
-import nextflow.script.params.TupleOutParam
+import nextflow.script.ProcessInputs
+import nextflow.script.ProcessOutputs
 
 import java.util.concurrent.atomic.AtomicLong
 
@@ -95,7 +89,7 @@ class DAG {
      * @param inputs The list of inputs entering in the process
      * @param outputs the list of outputs leaving the process
      */
-    void addProcessNode( String label, InputsList inputs, OutputsList outputs, TaskProcessor process=null ) {
+    void addProcessNode( String label, ProcessInputs inputs, ProcessOutputs outputs, TaskProcessor process=null ) {
         assert label
         assert inputs!=null
         assert outputs
@@ -234,30 +228,18 @@ class DAG {
 
     }
 
-    private List normalizeInputs( InputsList inputs ) {
+    private List normalizeInputs( ProcessInputs inputs ) {
 
-        inputs.collect { InParam p -> new ChannelHandler(channel: p.rawChannel, label: inputName0(p)) }
-
-    }
-
-    private String inputName0(InParam param) {
-        if( param instanceof TupleInParam ) return null
-        if( param instanceof EachInParam ) return null
-        return param.name
+        inputs.collect { p ->
+            new ChannelHandler(channel: p.getChannel(), label: p.getName())
+        }
     }
 
-    private List normalizeOutputs( OutputsList outputs ) {
+    private List normalizeOutputs( ProcessOutputs outputs ) {
 
-        def result = []
-        for(OutParam p :outputs) {
-            if( p instanceof DefaultOutParam )
-                break
-            final it = p.getOutChannel()
-            if( it!=null )
-                result << new ChannelHandler(channel: it, label: p instanceof TupleOutParam ? null : p.name)
+        outputs.collect { p ->
+            new ChannelHandler(channel: p.getChannel(), label: p.getName())
         }
-
-        return result
     }
 
     private List normalizeChannels( entry ) {
diff --git a/modules/nextflow/src/main/groovy/nextflow/dag/NodeMarker.groovy b/modules/nextflow/src/main/groovy/nextflow/dag/NodeMarker.groovy
index 68d56a12d5..a177000cec 100644
--- a/modules/nextflow/src/main/groovy/nextflow/dag/NodeMarker.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/dag/NodeMarker.groovy
@@ -21,8 +21,8 @@ import groovyx.gpars.dataflow.operator.DataflowProcessor
 import nextflow.Global
 import nextflow.Session
 import nextflow.processor.TaskProcessor
-import nextflow.script.params.InputsList
-import nextflow.script.params.OutputsList
+import nextflow.script.ProcessInputs
+import nextflow.script.ProcessOutputs
 /**
  * Helper class to mark DAG node with the proper labels
  *
@@ -46,7 +46,7 @@ class NodeMarker {
      * @param inputs The list of inputs entering in the process
      * @param outputs the list of outputs leaving the process
      */
-    static void addProcessNode( TaskProcessor process, InputsList inputs, OutputsList outputs ) {
+    static void addProcessNode( TaskProcessor process, ProcessInputs inputs, ProcessOutputs outputs ) {
         if( session && session.dag && !session.aborted )
             session.dag.addProcessNode( process.name, inputs, outputs, process )
     }
diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy
index 8013f97fea..490f5225e8 100644
--- a/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy
@@ -263,13 +263,12 @@ class PublishDir {
     /**
      * Apply the publishing process to the specified {@link TaskRun} instance
      *
-     * @param files Set of output files
      * @param task The task whose output need to be published
      */
     @CompileStatic
-    void apply( Set files, TaskRun task ) {
+    void apply( TaskRun task ) {
 
-        if( !files || !enabled )
+        if( !task.outputFiles || !enabled )
             return
 
         if( !path )
@@ -283,7 +282,7 @@ class PublishDir {
         this.stageInMode = task.config.stageInMode
         this.taskName = task.name
 
-        apply0(files)
+        apply0(task.outputFiles)
     }
 
 
diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskEnvCollector.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskEnvCollector.groovy
new file mode 100644
index 0000000000..f2740cc27f
--- /dev/null
+++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskEnvCollector.groovy
@@ -0,0 +1,58 @@
+/*
+ * Copyright 2013-2023, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package nextflow.processor
+
+import java.nio.file.Path
+
+import groovy.transform.CompileStatic
+/**
+ * Implements the collection of environment variables
+ * from the environment of a task execution.
+ *
+ * @author Paolo Di Tommaso 
+ * @author Ben Sherman 
+ */
+@CompileStatic
+class TaskEnvCollector {
+
+    private Path workDir
+
+    TaskEnvCollector(Path workDir) {
+        this.workDir = workDir
+    }
+
+    Map collect() {
+        final env = workDir.resolve(TaskRun.CMD_ENV).text
+        final result = new HashMap(50)
+        for( String line : env.readLines() ) {
+            def tokens = tokenize0(line)
+            def key = tokens[0]
+            def value = tokens[1]
+            if( !key )
+                continue
+            result.put(key, value)
+        }
+        return result
+    }
+
+    private List tokenize0(String line) {
+        int p = line.indexOf('=')
+        return p == -1
+                ? List.of(line, '')
+                : List.of(line.substring(0,p), line.substring(p+1))
+    }
+}
diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskFileCollector.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskFileCollector.groovy
new file mode 100644
index 0000000000..2219cc4c1c
--- /dev/null
+++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskFileCollector.groovy
@@ -0,0 +1,146 @@
+/*
+ * Copyright 2013-2023, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package nextflow.processor
+
+import java.nio.file.LinkOption
+import java.nio.file.Path
+import java.nio.file.NoSuchFileException
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import nextflow.exception.IllegalArityException
+import nextflow.exception.MissingFileException
+import nextflow.file.FileHelper
+import nextflow.file.FilePatternSplitter
+import nextflow.script.ProcessFileOutput
+/**
+ * Implements the collection of files from the work directory
+ * of a task execution.
+ *
+ * @author Paolo Di Tommaso 
+ * @author Ben Sherman 
+ */
+@Slf4j
+@CompileStatic
+class TaskFileCollecter {
+
+    private ProcessFileOutput param
+
+    private TaskRun task
+
+    TaskFileCollecter(ProcessFileOutput param, TaskRun task) {
+        this.param = param
+        this.task = task
+    }
+
+    Object collect() {
+        final List allFiles = []
+        final filePatterns = param.getFilePatterns(task.context, task.workDir)
+        boolean inputsExcluded = false
+
+        for( String filePattern : filePatterns ) {
+            List result = null
+
+            final splitter = param.glob ? FilePatternSplitter.glob().parse(filePattern) : null
+            if( splitter?.isPattern() ) {
+                result = fetchResultFiles(filePattern, task.workDir)
+                if( result && !param.includeInputs ) {
+                    result = excludeStagedInputs(task, result)
+                    log.trace "Process ${task.lazyName()} > after removing staged inputs: ${result}"
+                    inputsExcluded |= (result.size()==0)
+                }
+            }
+            else {
+                final path = param.glob ? splitter.strip(filePattern) : filePattern
+                final file = task.workDir.resolve(path)
+                final exists = checkFileExists(file)
+                if( exists )
+                    result = List.of(file)
+                else
+                    log.debug "Process `${task.lazyName()}` is unable to find [${file.class.simpleName}]: `$file` (pattern: `$filePattern`)"
+            }
+
+            if( result )
+                allFiles.addAll(result)
+
+            else if( !param.optional && (!param.arity || param.arity.min > 0) ) {
+                def msg = "Missing output file(s) `$filePattern` expected by process `${task.lazyName()}`"
+                if( inputsExcluded )
+                    msg += " (note: input files are not included in the default matching set)"
+                throw new MissingFileException(msg)
+            }
+        }
+
+        if( !param.isValidArity(allFiles.size()) )
+            throw new IllegalArityException("Incorrect number of output files for process `${task.lazyName()}` -- expected ${param.arity}, found ${allFiles.size()}")
+
+        return allFiles.size()==1 && param.isSingle() ? allFiles[0] : allFiles
+    }
+
+    /**
+     * Collect the file(s) matching the specified name or glob pattern
+     * in the given task work directory.
+     *
+     * @param pattern
+     * @param workDir
+     */
+    protected List fetchResultFiles(String pattern, Path workDir) {
+        final opts = [
+            relative: false,
+            hidden: param.hidden ?: pattern.startsWith('.'),
+            followLinks: param.followLinks,
+            maxDepth: param.maxDepth,
+            type: param.type ? param.type : ( pattern.contains('**') ? 'file' : 'any' )
+        ]
+
+        List files = []
+        try {
+            FileHelper.visitFiles(opts, workDir, pattern) { Path it -> files.add(it) }
+        }
+        catch( NoSuchFileException e ) {
+            throw new MissingFileException("Cannot access directory: '$workDir'", e)
+        }
+
+        return files.sort()
+    }
+
+    /**
+     * Remove each path in the given list whose name matches the name of
+     * an input file for the specified {@code TaskRun}
+     *
+     * @param task
+     * @param collectedFiles
+     */
+    protected List excludeStagedInputs(TaskRun task, List collectedFiles) {
+
+        final List allStagedFiles = task.getStagedInputs()
+        final List result = new ArrayList<>(collectedFiles.size())
+
+        for( int i = 0; i < collectedFiles.size(); i++ ) {
+            final file = collectedFiles.get(i)
+            final relativeName = task.workDir.relativize(file).toString()
+            if( !allStagedFiles.contains(relativeName) )
+                result.add(file)
+        }
+
+        return result
+    }
+
+    protected boolean checkFileExists(Path file) {
+        param.followLinks ? file.exists() : file.exists(LinkOption.NOFOLLOW_LINKS)
+    }
+}
diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy
index 29ded3970b..2a8a9622a7 100644
--- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy
@@ -19,13 +19,11 @@ import static nextflow.processor.ErrorStrategy.*
 
 import java.lang.reflect.InvocationTargetException
 import java.nio.file.FileSystems
-import java.nio.file.LinkOption
 import java.nio.file.NoSuchFileException
 import java.nio.file.Path
 import java.nio.file.Paths
 import java.util.concurrent.atomic.AtomicBoolean
 import java.util.concurrent.atomic.AtomicInteger
-import java.util.concurrent.atomic.AtomicIntegerArray
 import java.util.concurrent.atomic.LongAdder
 import java.util.regex.Matcher
 import java.util.regex.Pattern
@@ -75,7 +73,6 @@ import nextflow.extension.CH
 import nextflow.extension.DataflowHelper
 import nextflow.file.FileHelper
 import nextflow.file.FileHolder
-import nextflow.file.FilePatternSplitter
 import nextflow.file.FilePorter
 import nextflow.script.BaseScript
 import nextflow.script.BodyDef
@@ -84,21 +81,7 @@ import nextflow.script.ScriptMeta
 import nextflow.script.ScriptType
 import nextflow.script.TaskClosure
 import nextflow.script.bundle.ResourcesBundle
-import nextflow.script.params.DefaultOutParam
-import nextflow.script.params.EachInParam
-import nextflow.script.params.EnvInParam
-import nextflow.script.params.EnvOutParam
-import nextflow.script.params.FileInParam
 import nextflow.script.params.FileOutParam
-import nextflow.script.params.InParam
-import nextflow.script.params.MissingParam
-import nextflow.script.params.OptionalParam
-import nextflow.script.params.OutParam
-import nextflow.script.params.StdInParam
-import nextflow.script.params.StdOutParam
-import nextflow.script.params.TupleInParam
-import nextflow.script.params.TupleOutParam
-import nextflow.script.params.ValueInParam
 import nextflow.script.params.ValueOutParam
 import nextflow.util.ArrayBag
 import nextflow.util.BlankSeparatedList
@@ -397,14 +380,6 @@ class TaskProcessor {
         if ( !taskBody )
             throw new IllegalStateException("Missing task body for process `$name`")
 
-        /*
-         * Normalize the output
-         * - even though the output may be empty, let return the stdout as output by default
-         */
-        if ( config.getOutputs().size() == 0 ) {
-            config.fakeOutput()
-        }
-
         // the state agent
         state = new Agent<>(new StateObj(name))
         state.addListener { StateObj old, StateObj obj ->
@@ -434,14 +409,13 @@ class TaskProcessor {
          * When there is a single output channel, return let returns that item
          * otherwise return the list
          */
-        def result = config.getOutputs().channels
+        final result = config.getOutputs().getChannels()
         return result.size() == 1 ? result[0] : result
     }
 
     protected void createOperator(DataflowReadChannel source) {
         // determine whether the process is executed only once
         this.singleton = !CH.isChannelQueue(source)
-        config.getOutputs().setSingleton(singleton)
 
         // create inputs with control channel
         final control = CH.queue()
@@ -483,9 +457,6 @@ class TaskProcessor {
         // -- set the task instance as the current in this thread
         currentTask.set(task)
 
-        // -- validate input lengths
-        validateInputTuples(values)
-
         // -- map the inputs to a map and use to delegate closure values interpolation
         makeTaskContextStage1(task, values)
         makeTaskContextStage2(task)
@@ -515,28 +486,6 @@ class TaskProcessor {
         checkCachedOrLaunchTask(task, hash, resumable)
     }
 
-    @Memoized
-    private List getDeclaredInputTuple() {
-        getConfig().getInputs().ofType(TupleInParam)
-    }
-
-    protected void validateInputTuples( List values ) {
-
-        def declaredTuples = getDeclaredInputTuple()
-        for( int i=0; i
-            if( param instanceof TupleOutParam ) {
-                param.inner.each { task.setOutput(it) }
-            }
-            else
-                task.setOutput(param)
-        }
-
         return task
     }
 
@@ -807,16 +745,16 @@ class TaskProcessor {
         }
 
         try {
-            // -- expose task exit status to make accessible as output value
+            // -- set task properties in order to resolve outputs
+            task.workDir = folder
+            task.stdout = stdoutFile
             task.config.exitStatus = exitCode
             // -- check if all output resources are available
-            collectOutputs(task, folder, stdoutFile, task.context)
+            collectOutputs(task)
 
             // set the exit code in to the task object
             task.cached = true
             task.hash = hash
-            task.workDir = folder
-            task.stdout = stdoutFile
             if( exitCode != null ) {
                 task.exitStatus = exitCode
             }
@@ -834,6 +772,7 @@ class TaskProcessor {
             log.trace "[${safeTaskName(task)}] Missed cache > ${e.getMessage()} -- folder: $folder"
             task.exitStatus = Integer.MAX_VALUE
             task.workDir = null
+            task.stdout = null
             return false
         }
     }
@@ -1195,38 +1134,14 @@ class TaskProcessor {
      */
     @CompileStatic
     protected void publishOutputs( TaskRun task ) {
-        final publishList = task.config.getPublishDir()
-        if( !publishList ) {
-            return
-        }
+        final publishers = task.config.getPublishDir()
 
-        for( PublishDir pub : publishList ) {
-            publishOutputs0(task, pub)
-        }
-    }
+        for( PublishDir publisher : publishers ) {
+            if( publisher.overwrite == null )
+                publisher.overwrite = !task.cached
 
-    private void publishOutputs0( TaskRun task, PublishDir publish ) {
-
-        if( publish.overwrite == null ) {
-            publish.overwrite = !task.cached
+            publisher.apply(task)
         }
-
-        HashSet files = []
-        def outputs = task.getOutputsByType(FileOutParam)
-        for( Map.Entry entry : outputs ) {
-            final value = entry.value
-            if( value instanceof Path ) {
-                files.add((Path)value)
-            }
-            else if( value instanceof Collection ) {
-                files.addAll(value)
-            }
-            else if( value != null ) {
-                throw new IllegalArgumentException("Unknown output file object [${value.class.name}]: ${value}")
-            }
-        }
-
-        publish.apply(files, task)
     }
 
     /**
@@ -1235,46 +1150,12 @@ class TaskProcessor {
      */
     synchronized protected void bindOutputs( TaskRun task ) {
 
-        // -- creates the map of all tuple values to bind
-        Map tuples = [:]
-        for( OutParam param : config.getOutputs() ) {
-            tuples.put(param.index, [])
-        }
-
-        // -- collects the values to bind
-        for( OutParam param: task.outputs.keySet() ){
-            def value = task.outputs.get(param)
-
-            switch( param ) {
-            case StdOutParam:
-                log.trace "Process $name > normalize stdout param: $param"
-                value = value instanceof Path ? value.text : value?.toString()
-
-            case OptionalParam:
-                if( !value && param instanceof OptionalParam && param.optional ) {
-                    final holder = [] as MissingParam; holder.missing = param
-                    tuples[param.index] = holder
-                    break
-                }
-
-            case EnvOutParam:
-            case ValueOutParam:
-            case DefaultOutParam:
-                log.trace "Process $name > collecting out param: ${param} = $value"
-                tuples[param.index].add(value)
-                break
-
-            default:
-                throw new IllegalArgumentException("Illegal output parameter type: $param")
-            }
-        }
-
         // bind the output
         if( isFair0 ) {
-            fairBindOutputs0(tuples, task)
+            fairBindOutputs0(task.outputs, task)
         }
         else {
-            bindOutputs0(tuples)
+            bindOutputs0(task.outputs)
         }
 
         // -- finally prints out the task output when 'debug' is true
@@ -1283,7 +1164,7 @@ class TaskProcessor {
         }
     }
 
-    protected void fairBindOutputs0(Map emissions, TaskRun task) {
+    protected void fairBindOutputs0(List emissions, TaskRun task) {
         synchronized (isFair0) {
             // decrement -1 because tasks are 1-based
             final index = task.index-1
@@ -1305,287 +1186,36 @@ class TaskProcessor {
         }
     }
 
-    protected void bindOutputs0(Map tuples) {
+    protected void bindOutputs0(List outputs) {
         // -- bind out the collected values
-        for( OutParam param : config.getOutputs() ) {
-            final outValue = tuples[param.index]
-            if( outValue == null )
-                throw new IllegalStateException()
+        for( int i = 0; i < config.getOutputs().size(); i++ ) {
+            final param = config.getOutputs()[i]
+            final value = outputs[i]
 
-            if( outValue instanceof MissingParam ) {
-                log.debug "Process $name > Skipping output binding because one or more optional files are missing: $outValue.missing"
+            if( value == null ) {
+                log.debug "Process $name > Skipping output binding because one or more optional files are missing: ${param.name}"
                 continue
             }
 
-            log.trace "Process $name > Binding out param: ${param} = ${outValue}"
-            bindOutParam(param, outValue)
+            // clone collection values before emitting them so that the task processor
+            // can iterate over them without causing a race condition
+            // see https://github.com/nextflow-io/nextflow/issues/3768
+            log.trace "Process $name > Emitting output: ${param.name} = ${value}"
+            final copy = value instanceof Collection && value instanceof Cloneable ? value.clone() : value
+            param.getChannel().bind(copy)
         }
     }
 
-    protected void bindOutParam( OutParam param, List values ) {
-        log.trace "<$name> Binding param $param with $values"
-        final x = values.size() == 1 ? values[0] : values
-        final ch = param.getOutChannel()
-        if( ch != null ) {
-            // create a copy of the output list of operation made by a downstream task
-            // can modify the list which is used internally by the task processor
-            // and result in a potential error. See https://github.com/nextflow-io/nextflow/issues/3768
-            final copy = x instanceof List && x instanceof Cloneable ? x.clone() : x
-            // emit the final value
-            ch.bind(copy)
-        }
-    }
-
-    protected void collectOutputs( TaskRun task ) {
-        collectOutputs( task, task.getTargetDir(), task.@stdout, task.context )
-    }
-
     /**
      * Once the task has completed this method is invoked to collected all the task results
      *
      * @param task
      */
-    final protected void collectOutputs( TaskRun task, Path workDir, def stdout, Map context ) {
-        log.trace "<$name> collecting output: ${task.outputs}"
-
-        for( OutParam param : task.outputs.keySet() ) {
-
-            switch( param ) {
-                case StdOutParam:
-                    collectStdOut(task, (StdOutParam)param, stdout)
-                    break
-
-                case FileOutParam:
-                    collectOutFiles(task, (FileOutParam)param, workDir, context)
-                    break
-
-                case ValueOutParam:
-                    collectOutValues(task, (ValueOutParam)param, context)
-                    break
-
-                case EnvOutParam:
-                    collectOutEnvParam(task, (EnvOutParam)param, workDir)
-                    break
-
-                case DefaultOutParam:
-                    task.setOutput(param, DefaultOutParam.Completion.DONE)
-                    break
-
-                default:
-                    throw new IllegalArgumentException("Illegal output parameter: ${param.class.simpleName}")
-
-            }
-        }
-
-        // mark ready for output binding
+    protected void collectOutputs( TaskRun task ) {
+        task.outputs = config.getOutputs().collect( param -> param.resolve(task) )
         task.canBind = true
     }
 
-    protected void collectOutEnvParam(TaskRun task, EnvOutParam param, Path workDir) {
-
-        // fetch the output value
-        final val = collectOutEnvMap(workDir).get(param.name)
-        if( val == null && !param.optional )
-            throw new MissingValueException("Missing environment variable: $param.name")
-        // set into the output set
-        task.setOutput(param,val)
-        // trace the result
-        log.trace "Collecting param: ${param.name}; value: ${val}"
-
-    }
-
-    @Memoized(maxCacheSize = 10_000)
-    protected Map collectOutEnvMap(Path workDir) {
-        final env = workDir.resolve(TaskRun.CMD_ENV).text
-        final result = new HashMap(50)
-        for(String line : env.readLines() ) {
-            def (k,v) = tokenize0(line)
-            if (!k) continue
-            result.put(k,v)
-        }
-        return result
-    }
-
-    private List tokenize0(String line) {
-        int p=line.indexOf('=')
-        return p==-1
-                ? List.of(line,'')
-                : List.of(line.substring(0,p), line.substring(p+1))
-    }
-    
-    /**
-     * Collects the process 'std output'
-     *
-     * @param task The executed process instance
-     * @param param The declared {@link StdOutParam} object
-     * @param stdout The object holding the task produced std out object
-     */
-    protected void collectStdOut( TaskRun task, StdOutParam param, def stdout ) {
-
-        if( stdout == null && task.type == ScriptType.SCRIPTLET ) {
-            throw new IllegalArgumentException("Missing 'stdout' for process > ${safeTaskName(task)}")
-        }
-
-        if( stdout instanceof Path && !stdout.exists() ) {
-            throw new MissingFileException("Missing 'stdout' file: ${stdout.toUriString()} for process > ${safeTaskName(task)}")
-        }
-
-        task.setOutput(param, stdout)
-    }
-
-    protected void collectOutFiles( TaskRun task, FileOutParam param, Path workDir, Map context ) {
-
-        final List allFiles = []
-        // type file parameter can contain a multiple files pattern separating them with a special character
-        def entries = param.getFilePatterns(context, task.workDir)
-        boolean inputsRemovedFlag = false
-        // for each of them collect the produced files
-        for( String filePattern : entries ) {
-            List result = null
-
-            def splitter = param.glob ? FilePatternSplitter.glob().parse(filePattern) : null
-            if( splitter?.isPattern() ) {
-                result = fetchResultFiles(param, filePattern, workDir)
-                // filter the inputs
-                if( result && !param.includeInputs ) {
-                    result = filterByRemovingStagedInputs(task, result, workDir)
-                    log.trace "Process ${safeTaskName(task)} > after removing staged inputs: ${result}"
-                    inputsRemovedFlag |= (result.size()==0)
-                }
-            }
-            else {
-                def path = param.glob ? splitter.strip(filePattern) : filePattern
-                def file = workDir.resolve(path)
-                def exists = checkFileExists(file, param.followLinks)
-                if( exists )
-                    result = List.of(file)
-                else
-                    log.debug "Process `${safeTaskName(task)}` is unable to find [${file.class.simpleName}]: `$file` (pattern: `$filePattern`)"
-            }
-
-            if( result )
-                allFiles.addAll(result)
-
-            else if( !param.optional && (!param.arity || param.arity.min > 0) ) {
-                def msg = "Missing output file(s) `$filePattern` expected by process `${safeTaskName(task)}`"
-                if( inputsRemovedFlag )
-                    msg += " (note: input files are not included in the default matching set)"
-                throw new MissingFileException(msg)
-            }
-        }
-
-        if( !param.isValidArity(allFiles.size()) )
-            throw new IllegalArityException("Incorrect number of output files for process `${safeTaskName(task)}` -- expected ${param.arity}, found ${allFiles.size()}")
-
-        task.setOutput( param, allFiles.size()==1 && param.isSingle() ? allFiles[0] : allFiles )
-
-    }
-
-    protected boolean checkFileExists(Path file, boolean followLinks) {
-        followLinks ? file.exists() : file.exists(LinkOption.NOFOLLOW_LINKS)
-    }
-
-    protected void collectOutValues( TaskRun task, ValueOutParam param, Map ctx ) {
-
-        try {
-            // fetch the output value
-            final val = param.resolve(ctx)
-            // set into the output set
-            task.setOutput(param,val)
-            // trace the result
-            log.trace "Collecting param: ${param.name}; value: ${val}"
-        }
-        catch( MissingPropertyException e ) {
-            throw new MissingValueException("Missing value declared as output parameter: ${e.property}")
-        }
-
-    }
-
-    /**
-     * Collect the file(s) with the name specified, produced by the execution
-     *
-     * @param workDir The job working path
-     * @param namePattern The file name, it may include file name wildcards
-     * @return The list of files matching the specified name in lexicographical order
-     * @throws MissingFileException when no matching file is found
-     */
-    @PackageScope
-    List fetchResultFiles( FileOutParam param, String namePattern, Path workDir ) {
-        assert namePattern
-        assert workDir
-
-        List files = []
-        def opts = visitOptions(param, namePattern)
-        // scan to find the file with that name
-        try {
-            FileHelper.visitFiles(opts, workDir, namePattern) { Path it -> files.add(it) }
-        }
-        catch( NoSuchFileException e ) {
-            throw new MissingFileException("Cannot access directory: '$workDir'", e)
-        }
-
-        return files.sort()
-    }
-
-    /**
-     * Given a {@link FileOutParam} object create the option map for the
-     * {@link FileHelper#visitFiles(java.util.Map, java.nio.file.Path, java.lang.String, groovy.lang.Closure)} method
-     *
-     * @param param A task {@link FileOutParam}
-     * @param namePattern A file glob pattern
-     * @return A {@link Map} object holding the traverse options for the {@link FileHelper#visitFiles(java.util.Map, java.nio.file.Path, java.lang.String, groovy.lang.Closure)} method
-     */
-    @PackageScope
-    Map visitOptions( FileOutParam param, String namePattern ) {
-        final opts = [:]
-        opts.relative = false
-        opts.hidden = param.hidden ?: namePattern.startsWith('.')
-        opts.followLinks = param.followLinks
-        opts.maxDepth = param.maxDepth
-        opts.type = param.type ? param.type : ( namePattern.contains('**') ? 'file' : 'any' )
-        return opts
-    }
-
-    /**
-     * Given a list of {@code Path} removes all the hidden file i.e. the ones which names starts with a dot char
-     * @param files A list of {@code Path}
-     * @return The result list not containing hidden file entries
-     */
-    @PackageScope
-    List filterByRemovingHiddenFiles( List files ) {
-        files.findAll { !it.getName().startsWith('.') }
-    }
-
-    /**
-     * Given a list of {@code Path} removes all the entries which name match the name of
-     * file used as input for the specified {@code TaskRun}
-     *
-     * See TaskRun#getStagedInputs
-     *
-     * @param task
-     *      A {@link TaskRun} object representing the task executed
-     * @param collectedFiles
-     *      Collection of candidate output files
-     * @return
-     *      List of the actual output files (not including any input matching an output file name pattern)
-     */
-    @PackageScope
-    List filterByRemovingStagedInputs( TaskRun task, List collectedFiles, Path workDir ) {
-
-        // get the list of input files
-        final List allStaged = task.getStagedInputs()
-        final List result = new ArrayList<>(collectedFiles.size())
-
-        for( int i=0; i()
         int count = 0
 
         final FilePorter.Batch batch = session.filePorter.newBatch(executor.getStageDir())
 
         // -- resolve input files against the task context
-        for( def fileInput : allFileInputs ) {
-            final normalized = normalizeInputToFiles(fileInput.getValue(ctx), count, fileInput.isPathQualifier(), batch)
-            final resolved = expandWildcards( fileInput.getFilePattern(ctx), normalized )
+        for( def param : config.getInputs().files ) {
+            final val = param.getValue(ctx)
+            final normalized = normalizeInputToFiles(val, count, param.isPathQualifier(), batch)
+            final resolved = expandWildcards( param.getFilePattern(ctx), normalized )
 
-            if( !fileInput.isValidArity(resolved.size()) )
+            if( !param.isValidArity(resolved.size()) )
                 throw new IllegalArityException("Incorrect number of input files for process `${safeTaskName(task)}` -- expected ${param.arity}, found ${resolved.size()}")
 
-            ctx.put( fileInput.getName(ctx), singleItemOrList(resolved, fileInput.isSingle(), task.type) )
+            // add to context if the path was declared with a variable name
+            if( param.name )
+                ctx.put( param.name, singleItemOrList(resolved, param.isSingle(), task.type) )
+
             count += resolved.size()
+
             for( FileHolder item : resolved ) {
                 Integer num = allNames.getOrCreate(item.stageName, 0) +1
                 allNames.put(item.stageName,num)
@@ -1991,7 +1590,7 @@ class TaskProcessor {
             keys << task.getContainerFingerprint()
 
         // add task inputs
-        keys.add( task.config.get('vals') )
+        keys.add( task.config.get('vars') )
         keys.add( task.inputFiles )
         keys.add( task.getInputEnvironment() )
         keys.add( task.stdin )
diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy
index 57f592edad..020e070252 100644
--- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy
@@ -22,6 +22,7 @@ import java.nio.file.Path
 import java.util.concurrent.ConcurrentHashMap
 
 import com.google.common.hash.HashCode
+import groovy.transform.Memoized
 import groovy.transform.PackageScope
 import groovy.util.logging.Slf4j
 import nextflow.Session
@@ -38,7 +39,6 @@ import nextflow.script.BodyDef
 import nextflow.script.ScriptType
 import nextflow.script.TaskClosure
 import nextflow.script.bundle.ResourcesBundle
-import nextflow.script.params.EnvOutParam
 import nextflow.script.params.FileOutParam
 import nextflow.script.params.OutParam
 import nextflow.script.params.ValueOutParam
@@ -86,15 +86,18 @@ class TaskRun implements Cloneable {
     List inputFiles = []
 
     /**
-     * Holds the output value(s) for each task output parameter
+     * The list of resolved output files
+     *
+     * @see ProcessOutput#path(String)
      */
-    Map outputs = [:]
-
+    Set outputFiles = []
 
-    void setOutput( OutParam param, Object value = null ) {
-        assert param
-        outputs[param] = value
-    }
+    /**
+     * The list of resolved task outputs
+     *
+     * @see TaskProcessor#collectOutputs(TaskRun)
+     */
+    List outputs = []
 
 
     /**
@@ -414,44 +417,22 @@ class TaskRun implements Cloneable {
     Map getInputFilesMap() {
         def result = [:]
         for( FileHolder it : inputFiles )
-            result[ it.stageName ] = it.storePath
+            result.put(it.stageName, it.storePath)
         return result
     }
 
     /**
-     * Look at the {@code nextflow.script.FileOutParam} which name is the expected
-     *  output name
-     *
+     * Get the list of expected output file patterns.
      */
+    @Memoized
     List getOutputFilesNames() {
-        cache0.computeIfAbsent('outputFileNames', (it)-> getOutputFilesNames0())
-    }
-
-    private List getOutputFilesNames0() {
-        def result = []
-
-        for( FileOutParam param : getOutputsByType(FileOutParam).keySet() ) {
+        final declaredOutputs = processor.config.getOutputs()
+        final result = []
+        for( def param : declaredOutputs.files.values() )
             result.addAll( param.getFilePatterns(context, workDir) )
-        }
-
         return result.unique()
     }
 
-    /**
-     * Get the map of *output* objects by the given {@code OutParam} type
-     *
-     * @param types One or more subclass of {@code OutParam}
-     * @return An associative array containing all the objects for the specified type
-     */
-    def  Map getOutputsByType( Class... types ) {
-        def result = [:]
-        for( def it : outputs ) {
-            if( types.contains(it.key.class) )
-                result << it
-        }
-        return result
-    }
-
     /**
      * @return A map containing the task environment defined as input declaration by this task
      */
@@ -544,8 +525,8 @@ class TaskRun implements Cloneable {
     }
 
     List getOutputEnvNames() {
-        final items = getOutputsByType(EnvOutParam)
-        return items ? new ArrayList(items.keySet()*.name) : Collections.emptyList()
+        final declaredOutputs = processor.getConfig().getOutputs()
+        return new ArrayList(declaredOutputs.env.values())
     }
 
     Path getCondaEnv() {
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy b/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy
index 0c4b7b8918..2b69237bae 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy
@@ -28,7 +28,9 @@ import nextflow.ast.ProcessFn
 import nextflow.ast.WorkflowFn
 import nextflow.exception.AbortOperationException
 import nextflow.script.dsl.ProcessBuilder
+import nextflow.script.dsl.ProcessDsl
 import nextflow.script.dsl.ProcessInputsBuilder
+import nextflow.script.dsl.ProcessOutputsBuilder
 import nextflow.script.dsl.WorkflowBuilder
 /**
  * Any user defined script will extends this class, it provides the base execution context
@@ -103,7 +105,7 @@ abstract class BaseScript extends Script implements ExecutionContext {
      * @param rawBody
      */
     protected void process(String name, Closure rawBody) {
-        final builder = new ProcessBuilder(this, name)
+        final builder = new ProcessDsl(this, name)
         final copy = (Closure)rawBody.clone()
         copy.delegate = builder
         copy.resolveStrategy = Closure.DELEGATE_FIRST
@@ -200,30 +202,35 @@ abstract class BaseScript extends Script implements ExecutionContext {
 
         // build process from annotation
         final builder = new ProcessBuilder(this, name)
-        final inputsBuilder = new ProcessInputsBuilder(builder.getConfig())
 
-        applyDsl(inputsBuilder, processFn.inputs())
+        // -- directives
         applyDsl(builder, processFn.directives())
-        applyDsl(builder, processFn.outputs())
 
-        // get method parameters
-        builder.withParams(processFn.params())
+        // -- inputs
+        final inputs = new ProcessInputsBuilder()
+        applyDsl(inputs, processFn.inputs())
 
-        // determine process type
-        def type
-        if( processFn.script() )
-            type = 'script'
-        else if( processFn.shell() )
-            type = 'shell'
-        else
-            type = 'exec'
+        for( String param : processFn.params() )
+            inputs.take(param)
+
+        // -- outputs
+        final outputs = new ProcessOutputsBuilder()
+        applyDsl(outputs, processFn.outputs())
 
-        // create task body
-        final taskBody = new BodyDef( this.&"${name}", processFn.source(), type, [] )
-        builder.withBody(taskBody)
+        // -- process type
+        final type =
+            processFn.script() ? 'script'
+            : processFn.shell() ? 'shell'
+            : 'exec'
+
+        final process = builder
+            .withInputs(inputs.build())
+            .withOutputs(outputs.build())
+            .withBody(this.&"${name}", type, processFn.source())
+            .build()
 
         // register process
-        meta.addDefinition(builder.build())
+        meta.addDefinition(process)
     }
 
     private void registerWorkflowFn(Method method) {
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ChannelOut.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ChannelOut.groovy
index 1d48e150c4..0187d59f9e 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ChannelOut.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ChannelOut.groovy
@@ -21,8 +21,6 @@ import groovy.util.logging.Slf4j
 import groovyx.gpars.dataflow.DataflowWriteChannel
 import nextflow.ast.DslCodeVisitor
 import nextflow.exception.DuplicateChannelNameException
-import nextflow.script.params.OutParam
-import nextflow.script.params.OutputsList
 /**
  * Models the output of a process or a workflow component returning
  * more than one output channels
@@ -52,12 +50,12 @@ class ChannelOut implements List {
         this.channels = Collections.unmodifiableMap(new LinkedHashMap(channels))
     }
 
-    ChannelOut(OutputsList outs) {
+    ChannelOut(ProcessOutputs outs) {
         channels = new HashMap<>(outs.size())
         final onlyWithName = new ArrayList(outs.size())
-        for( OutParam param : outs ) {
-            final ch = param.getOutChannel()
-            final name = param.channelEmitName
+        for( ProcessOutput param : outs ) {
+            final ch = param.getChannel()
+            final name = param.getName()
             onlyWithName.add(ch)
             if(name) {
                 if(channels.containsKey(name)) throw new DuplicateChannelNameException("Output channel name `$name` is used more than one time")
diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/PathArityAware.groovy b/modules/nextflow/src/main/groovy/nextflow/script/PathArityAware.groovy
similarity index 88%
rename from modules/nextflow/src/main/groovy/nextflow/processor/PathArityAware.groovy
rename to modules/nextflow/src/main/groovy/nextflow/script/PathArityAware.groovy
index 59c1b5bfac..6587619981 100644
--- a/modules/nextflow/src/main/groovy/nextflow/processor/PathArityAware.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/PathArityAware.groovy
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package nextflow.processor
+package nextflow.script
 
 import groovy.transform.CompileStatic
 import groovy.transform.EqualsAndHashCode
@@ -32,9 +32,14 @@ trait PathArityAware {
 
     Range getArity() { arity }
 
-    def setArity(String value) {
+    def setArity(Object value) {
+        if( value !instanceof String )
+            throw new IllegalArityException("Path arity should be a string number (e.g. '1') or range (e.g. '1..*')")
+        
+        value = (String)value
+
         if( value.isInteger() ) {
-            def n = value.toInteger()
+            final n = value.toInteger()
             this.arity = new Range(n, n)
             return this
         }
@@ -52,7 +57,7 @@ trait PathArityAware {
             }
         }
 
-        throw new IllegalArityException("Path arity should be a number (e.g. '1') or a range (e.g. '1..*')")
+        throw new IllegalArityException("Path arity should be a string number (e.g. '1') or range (e.g. '1..*')")
     }
 
     /**
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessConfig.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessConfig.groovy
index 4dc4e2f483..e6b8fbeae1 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessConfig.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessConfig.groovy
@@ -23,9 +23,6 @@ import nextflow.executor.BashWrapperBuilder
 import nextflow.processor.ErrorStrategy
 import nextflow.processor.TaskConfig
 import static nextflow.util.CacheHelper.HashMode
-import nextflow.script.params.DefaultOutParam
-import nextflow.script.params.InputsList
-import nextflow.script.params.OutputsList
 
 /**
  * Holds the process configuration properties
@@ -65,20 +62,15 @@ class ProcessConfig implements Map, Cloneable {
      */
     private String processName
 
-    /**
-     * List of parameter names defined by a process function.
-     */
-    private String[] params
-
     /**
      * List of process input definitions
      */
-    private InputsList inputs = new InputsList()
+    private ProcessInputs inputs
 
     /**
      * List of process output definitions
      */
-    private OutputsList outputs = new OutputsList()
+    private ProcessOutputs outputs
 
     protected ProcessConfig( BaseScript script ) {
         ownerScript = script
@@ -119,8 +111,14 @@ class ProcessConfig implements Map, Cloneable {
     }
 
     @PackageScope
-    ProcessConfig setParams(String[] params) {
-        this.params = params
+    ProcessConfig setInputs(ProcessInputs inputs) {
+        this.inputs = inputs
+        return this
+    }
+
+    @PackageScope
+    ProcessConfig setOutputs(ProcessOutputs outputs) {
+        this.outputs = outputs
         return this
     }
 
@@ -128,9 +126,6 @@ class ProcessConfig implements Map, Cloneable {
     Object getProperty( String name ) {
 
         switch( name ) {
-            case 'params':
-                return getParams()
-
             case 'inputs':
                 return getInputs()
 
@@ -165,22 +160,14 @@ class ProcessConfig implements Map, Cloneable {
         return new TaskConfig(configProperties)
     }
 
-    String[] getParams() {
-        params
-    }
-
-    InputsList getInputs() {
+    ProcessInputs getInputs() {
         inputs
     }
 
-    OutputsList getOutputs() {
+    ProcessOutputs getOutputs() {
         outputs
     }
 
-    void fakeOutput() {
-        new DefaultOutParam(this)
-    }
-
     boolean isCacheable() {
         def value = configProperties.cache
         if( value == null )
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy
index 22c47da095..b2162267cd 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy
@@ -18,7 +18,6 @@ package nextflow.script
 
 import groovy.transform.CompileStatic
 import groovy.util.logging.Slf4j
-import groovyx.gpars.dataflow.DataflowBroadcast
 import groovyx.gpars.dataflow.DataflowReadChannel
 import nextflow.Const
 import nextflow.Global
@@ -26,13 +25,7 @@ import nextflow.Session
 import nextflow.exception.ScriptRuntimeException
 import nextflow.extension.CH
 import nextflow.extension.CombineOp
-import nextflow.extension.MergeOp
 import nextflow.script.dsl.ProcessBuilder
-import nextflow.script.params.BaseInParam
-import nextflow.script.params.BaseOutParam
-import nextflow.script.params.EachInParam
-import nextflow.script.params.InputsList
-import nextflow.script.params.OutputsList
 
 /**
  * Models a nextflow process definition
@@ -118,9 +111,9 @@ class ProcessDef extends BindableDef implements IterableDef, ChainableDef {
         return result
     }
 
-    private InputsList getDeclaredInputs() { config.getInputs() }
+    private ProcessInputs getDeclaredInputs() { config.getInputs() }
 
-    private OutputsList getDeclaredOutputs() { config.getOutputs() }
+    private ProcessOutputs getDeclaredOutputs() { config.getOutputs() }
 
     BaseScript getOwner() { owner }
 
@@ -152,30 +145,12 @@ class ProcessDef extends BindableDef implements IterableDef, ChainableDef {
         // initialise process config
         initialize()
 
-        // create merged input channel
+        // create input channel
         final source = collectInputs(args)
 
         // set output channels
-        // note: the result object must be an array instead of a List to allow process
-        // composition ie. to use the process output as the input in another process invocation
-        if( declaredOutputs.size() ) {
-            final allScalarValues = declaredInputs.allScalarInputs()
-            final hasEachParams = declaredInputs.any { it instanceof EachInParam }
-            final singleton = allScalarValues && !hasEachParams
-
-            // check for feedback channels
-            final feedbackChannels = getFeedbackChannels()
-            if( feedbackChannels && feedbackChannels.size() != declaredOutputs.size() )
-                throw new ScriptRuntimeException("Process `$processName` inputs and outputs do not have the same cardinality - Feedback loop is not supported"  )
-
-            for(int i=0; i0, "Process output should contains at least one channel"
-        return output = new ChannelOut(copyOuts)
+        // note: the result object must be an array instead of a List to allow process
+        // composition ie. to use the process output as the input in another process invocation
+        return output = new ChannelOut(declaredOutputs)
     }
 
     private DataflowReadChannel collectInputs(Object[] args0) {
         final args = ChannelOut.spread(args0)
-        final hasDeclaredInputs = config.getParams()==null
-        if( hasDeclaredInputs && args.size() != declaredInputs.size() )
+        if( args.size() != declaredInputs.size() )
             throw new ScriptRuntimeException(missMatchErrMessage(processName, declaredInputs.size(), args.size()))
 
         // emit value channel if process has no inputs
@@ -207,21 +182,16 @@ class ProcessDef extends BindableDef implements IterableDef, ChainableDef {
         }
 
         // set input channels
-        for( int i = 0; i < declaredInputs.size(); i++ ) {
-            final param = (declaredInputs[i] as BaseInParam)
-            param.setFrom(args[i])
-            param.init()
-        }
+        for( int i = 0; i < declaredInputs.size(); i++ )
+            declaredInputs[i].bind(args[i])
 
         // normalize args into channels
-        final inputs = hasDeclaredInputs
-            ? declaredInputs.getChannels()
-            : args.collect(ch -> getInChannel(ch))
+        final inputs = declaredInputs.getChannels()
 
         // make sure no more than one queue channel is provided
         int count = 0
         for( int i = 0; i < inputs.size(); i++ )
-            if( CH.isChannelQueue(inputs[i]) && (!hasDeclaredInputs || declaredInputs[i] !instanceof EachInParam) )
+            if( CH.isChannelQueue(inputs[i]) )
                 count += 1
 
         if( count > 1 )
@@ -239,26 +209,22 @@ class ProcessDef extends BindableDef implements IterableDef, ChainableDef {
         return result
     }
 
-    private DataflowReadChannel getInChannel(Object obj) {
-        if( obj == null )
-            throw new IllegalArgumentException('A process input channel evaluates to null')
-
-        final result = obj instanceof Closure
-            ? obj.call()
-            : obj
+    private void collectOutputs(boolean singleton) {
+        // emit stdout if no outputs are defined
+        if( declaredOutputs.size() == 0 ) {
+            declaredOutputs.setDefault()
+            return
+        }
 
-        if( result == null )
-            throw new IllegalArgumentException('A process input channel evaluates to null')
+        // check for feedback channels
+        final feedbackChannels = getFeedbackChannels()
+        if( feedbackChannels && feedbackChannels.size() != declaredOutputs.size() )
+            throw new ScriptRuntimeException("Process `$processName` inputs and outputs do not have the same cardinality - Feedback loop is not supported"  )
 
-        def inChannel
-        if ( result instanceof DataflowReadChannel || result instanceof DataflowBroadcast )
-            inChannel = CH.getReadChannel(result)
-        else {
-            inChannel = CH.value()
-            inChannel.bind(result)
+        for(int i=0; i)body.clone()
diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskFileInput.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessFileInput.groovy
similarity index 72%
rename from modules/nextflow/src/main/groovy/nextflow/processor/TaskFileInput.groovy
rename to modules/nextflow/src/main/groovy/nextflow/script/ProcessFileInput.groovy
index a1e8909cce..2f6488a64e 100644
--- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskFileInput.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessFileInput.groovy
@@ -14,49 +14,51 @@
  * limitations under the License.
  */
 
-package nextflow.processor
+package nextflow.script
 
 import groovy.transform.CompileStatic
 
 /**
- * Models a file input directive, which defines the file
- * or set of files to be staged into the task environment.
+ * Models a process file input, which defines a file
+ * or set of files to be staged into a task work directory.
  *
  * @author Ben Sherman 
  */
 @CompileStatic
-class TaskFileInput implements PathArityAware {
+class ProcessFileInput implements PathArityAware {
+
     private Object value
-    private boolean coerceToPath
+
     private String name
+
+    private boolean coerceToPath
+
     private Object filePattern
 
-    TaskFileInput(Object value, boolean coerceToPath, String name, Map opts) {
+    ProcessFileInput(Object value, String name, boolean coerceToPath, Map opts) {
         this.value = value
-        this.coerceToPath = coerceToPath
         this.name = name
+        this.coerceToPath = coerceToPath
         this.filePattern = opts.stageAs ?: opts.name
 
-        if( opts.arity )
-            this.setArity(opts.arity.toString())
+        for( Map.Entry entry : opts )
+            setProperty(entry.key, entry.value)
+    }
+
+    void setStageAs(String value) {
+        this.filePattern = value
     }
 
     Object getValue(Map ctx) {
         return resolve(ctx, value)
     }
 
-    boolean isPathQualifier() {
-        return coerceToPath
+    String getName() {
+        return name
     }
 
-    String getName(Map ctx) {
-        if( name != null )
-            return name
-
-        if( value != null )
-            return resolve(ctx, value)
-
-        return null
+    boolean isPathQualifier() {
+        return coerceToPath
     }
 
     String getFilePattern(Map ctx) {
@@ -66,7 +68,7 @@ class TaskFileInput implements PathArityAware {
         return filePattern = '*'
     }
 
-    private Object resolve( Map ctx, value ) {
+    protected Object resolve(Map ctx, Object value) {
         if( value instanceof GString )
             return value.cloneAsLazy(ctx)
 
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessFileOutput.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessFileOutput.groovy
new file mode 100644
index 0000000000..92e4c15f81
--- /dev/null
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessFileOutput.groovy
@@ -0,0 +1,140 @@
+/*
+ * Copyright 2013-2023, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package nextflow.script
+
+import java.nio.file.Path
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import nextflow.exception.IllegalFileException
+import nextflow.file.FilePatternSplitter
+import nextflow.util.BlankSeparatedList
+/**
+ * Models a process file output, which defines a file
+ * or set of files to be unstaged from a task work directory.
+ *
+ * @author Paolo Di Tommaso 
+ * @author Ben Sherman 
+ */
+@Slf4j
+@CompileStatic
+class ProcessFileOutput implements PathArityAware {
+
+    private Object target
+
+    /**
+     * When true it will not fail if no files are found.
+     */
+    boolean optional
+
+    /**
+     * When true it follows symbolic links during directories tree traversal, otherwise they are managed as files (default: true)
+     */
+    boolean followLinks = true
+
+    /**
+     * When true the specified name is interpreted as a glob pattern (default: true)
+     */
+    boolean glob = true
+
+    /**
+     * When {@code true} star wildcard (*) matches hidden files (files starting with a dot char)
+     * By default it does not, coherently with linux bash rule
+     */
+    boolean hidden
+
+    /**
+     * When {@code true} file pattern includes input files as well as output files.
+     * By default a file pattern matches only against files produced by the process, not
+     * the ones received as input
+     */
+    boolean includeInputs
+
+    /**
+     * Maximum number of directory levels to visit (default: no limit)
+     */
+    Integer maxDepth
+
+    /**
+     * The type of path to output, either 'file', 'dir' or 'any'
+     */
+    String type
+
+    ProcessFileOutput(Object target, Map opts) {
+        this.target = target
+
+        for( Map.Entry entry : opts )
+            setProperty(entry.key, entry.value)
+    }
+
+    List getFilePatterns(Map context, Path workDir) {
+        final entry = resolve(context, target)
+
+        if( !entry )
+            return []
+
+        // -- single path
+        if( entry instanceof Path )
+            return [ relativize(entry, workDir) ]
+
+        // -- multiple paths
+        if( entry instanceof BlankSeparatedList || entry instanceof List )
+            return entry.collect( path -> relativize(path.toString(), workDir) )
+
+        // -- literal file name
+        return [ relativize(entry.toString(), workDir) ]
+    }
+
+    protected Object resolve(Map ctx, Object value) {
+
+        if( value instanceof GString )
+            return value.cloneAsLazy(ctx)
+
+        if( value instanceof Closure )
+            return ctx.with(value)
+
+        return value.toString()
+    }
+
+    protected String relativize(String path, Path workDir) {
+        if( !path.startsWith('/') )
+            return path
+
+        final dir = workDir.toString()
+        if( !path.startsWith(dir) )
+            throw new IllegalFileException("File `$path` is outside the scope of the process work directory: $workDir")
+
+        if( path.length()-dir.length()<2 )
+            throw new IllegalFileException("Missing output file name")
+
+        return path.substring(dir.size()+1)
+    }
+
+    protected String relativize(Path path, Path workDir) {
+        if( !path.isAbsolute() )
+            return glob ? FilePatternSplitter.GLOB.escape(path) : path
+
+        if( !path.startsWith(workDir) )
+            throw new IllegalFileException("File `$path` is outside the scope of the process work directory: $workDir")
+
+        if( path.nameCount == workDir.nameCount )
+            throw new IllegalFileException("Missing output file name")
+
+        final rel = path.subpath(workDir.getNameCount(), path.getNameCount())
+        return glob ? FilePatternSplitter.GLOB.escape(rel) : rel
+    }
+}
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessInput.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessInput.groovy
new file mode 100644
index 0000000000..097eb3f79a
--- /dev/null
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessInput.groovy
@@ -0,0 +1,78 @@
+/*
+ * Copyright 2013-2023, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package nextflow.script
+
+import groovy.transform.CompileStatic
+import groovyx.gpars.dataflow.DataflowBroadcast
+import groovyx.gpars.dataflow.DataflowReadChannel
+import nextflow.extension.CH
+
+/**
+ * Models a process input.
+ *
+ * @author Ben Sherman 
+ */
+@CompileStatic
+class ProcessInput implements Cloneable {
+
+    private String name
+
+    private Object arg
+
+    private DataflowReadChannel channel
+
+    ProcessInput(String name) {
+        this.name = name
+    }
+
+    String getName() {
+        return name
+    }
+
+    void bind(Object arg) {
+        this.arg = arg
+        this.channel = getInChannel(arg)
+    }
+
+    private DataflowReadChannel getInChannel(Object obj) {
+        if( obj == null )
+            throw new IllegalArgumentException('A process input channel evaluates to null')
+
+        final result = obj instanceof Closure
+            ? obj.call()
+            : obj
+
+        if( result == null )
+            throw new IllegalArgumentException('A process input channel evaluates to null')
+
+        def inChannel
+        if ( result instanceof DataflowReadChannel || result instanceof DataflowBroadcast ) {
+            inChannel = CH.getReadChannel(result)
+        }
+        else {
+            inChannel = CH.value()
+            inChannel.bind(result)
+        }
+
+        return inChannel
+    }
+
+    DataflowReadChannel getChannel() {
+        return channel
+    }
+
+}
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessInputs.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessInputs.groovy
new file mode 100644
index 0000000000..cee0aa7338
--- /dev/null
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessInputs.groovy
@@ -0,0 +1,55 @@
+/*
+ * Copyright 2013-2023, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package nextflow.script
+
+import groovyx.gpars.dataflow.DataflowReadChannel
+
+/**
+ * Models the process inputs.
+ *
+ * @author Ben Sherman 
+ */
+class ProcessInputs implements List, Cloneable {
+
+    @Delegate
+    private List target = []
+
+    Map env = [:]
+
+    List files = []
+
+    Object stdin
+
+    @Override
+    ProcessInputs clone() {
+        def result = (ProcessInputs)super.clone()
+        result.target = new ArrayList<>(target.size())
+        for( ProcessInput param : target ) {
+            result.target.add((ProcessInput)param.clone())
+        }
+        return result
+    }
+
+    List getNames() {
+        return target*.getName()
+    }
+
+    List getChannels() {
+        return target*.getChannel()
+    }
+
+}
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessOutput.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessOutput.groovy
new file mode 100644
index 0000000000..4f7df7e302
--- /dev/null
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessOutput.groovy
@@ -0,0 +1,172 @@
+/*
+ * Copyright 2013-2023, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package nextflow.script
+
+import java.nio.file.Path
+
+import groovy.transform.CompileStatic
+import groovy.transform.Memoized
+import groovy.util.logging.Slf4j
+import groovyx.gpars.dataflow.DataflowWriteChannel
+import nextflow.exception.MissingFileException
+import nextflow.exception.MissingValueException
+import nextflow.processor.TaskEnvCollector
+import nextflow.processor.TaskFileCollecter
+import nextflow.processor.TaskRun
+/**
+ * Models a process output.
+ *
+ * @author Ben Sherman 
+ */
+@Slf4j
+@CompileStatic
+class ProcessOutput implements Cloneable {
+
+    static enum Shortcuts { STDOUT }
+
+    private ProcessOutputs parent
+
+    private Object target
+
+    private String name
+
+    private boolean optional
+
+    private DataflowWriteChannel channel
+
+    ProcessOutput(ProcessOutputs parent, Object target, Map opts) {
+        this.parent = parent
+        this.target = target
+
+        if( opts.name )
+            this.name = opts.name
+        if( opts.optional )
+            this.optional = true
+    }
+
+    String getName() {
+        return name
+    }
+
+    void setChannel(DataflowWriteChannel channel) {
+        this.channel = channel
+    }
+
+    DataflowWriteChannel getChannel() {
+        return channel
+    }
+
+    Object resolve(TaskRun task) {
+        final ctx = new ResolverContext(parent, optional, task)
+        return resolve0(ctx)
+    }
+
+    private Object resolve0(ResolverContext ctx) {
+        if( target == Shortcuts.STDOUT )
+            return ctx.stdout()
+
+        if( target instanceof Closure )
+            return ctx.with(target)
+
+        return target
+    }
+
+    static private class ResolverContext {
+
+        private ProcessOutputs parent
+
+        private boolean optional
+
+        private TaskRun task
+
+        ResolverContext(ProcessOutputs parent, boolean optional, TaskRun task) {
+            this.parent = parent
+            this.optional = optional
+            this.task = task
+        }
+
+        /**
+         * Get an environment variable from the task environment.
+         *
+         * @param key
+         */
+        String env(String key) {
+            final varName = parent.env.get(key)
+            final result = env0(task.workDir).get(varName)
+
+            if( result == null && !optional )
+                throw new MissingValueException("Missing environment variable: $varName")
+
+            return result
+        }
+
+        @Memoized
+        static private Map env0(Path workDir) {
+            new TaskEnvCollector(workDir).collect()
+        }
+
+        /**
+         * Get a file or list of files from the task environment.
+         *
+         * @param key
+         */
+        Object path(String key) {
+            final param = parent.files.get(key)
+            final result = new TaskFileCollecter(param, task).collect()
+
+            if( result instanceof Path )
+                task.outputFiles.add(result)
+            else if( result instanceof Collection )
+                task.outputFiles.addAll(result)
+
+            return result
+        }
+
+        /**
+         * Get the standard output from the task environment.
+         */
+        Object stdout() {
+            final result = task.@stdout
+
+            if( result == null && task.type == ScriptType.SCRIPTLET )
+                throw new IllegalArgumentException("Missing 'stdout' for process > ${task.lazyName()}")
+
+            if( result instanceof Path && !result.exists() )
+                throw new MissingFileException("Missing 'stdout' file: ${result.toUriString()} for process > ${task.lazyName()}")
+
+            return result
+        }
+
+        /**
+         * Get a variable from the task context.
+         *
+         * @param name
+         */
+        @Override
+        Object getProperty(String name) {
+            if( name == 'stdout' )
+                return stdout()
+
+            try {
+                return task.context.get(name)
+            }
+            catch( MissingPropertyException e ) {
+                throw new MissingValueException("Missing variable in emit statement: ${e.property}")
+            }
+        }
+    }
+}
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessOutputs.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessOutputs.groovy
new file mode 100644
index 0000000000..97e2ab30a9
--- /dev/null
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessOutputs.groovy
@@ -0,0 +1,59 @@
+/*
+ * Copyright 2013-2023, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package nextflow.script
+
+import groovyx.gpars.dataflow.DataflowQueue
+import groovyx.gpars.dataflow.DataflowWriteChannel
+
+/**
+ * Models the process outputs.
+ *
+ * @author Ben Sherman 
+ */
+class ProcessOutputs implements List, Cloneable {
+
+    @Delegate
+    private List target = []
+
+    Map env = [:]
+
+    Map files = [:]
+
+    @Override
+    ProcessOutputs clone() {
+        def result = (ProcessOutputs)super.clone()
+        result.target = new ArrayList<>(target.size())
+        for( ProcessOutput param : target )
+            result.add((ProcessOutput)param.clone())
+        return result
+    }
+
+    List getNames() {
+        return target*.getName()
+    }
+
+    List getChannels() {
+        return target*.getChannel()
+    }
+
+    void setDefault() {
+        final param = new ProcessOutput(ProcessOutput.Shortcuts.STDOUT, [:])
+        param.setChannel(new DataflowQueue())
+        target.add(param)
+    }
+
+}
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessBuilder.groovy b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessBuilder.groovy
index 710be1eef7..63e134a739 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessBuilder.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessBuilder.groovy
@@ -26,14 +26,15 @@ import nextflow.exception.IllegalDirectiveException
 import nextflow.exception.ScriptRuntimeException
 import nextflow.processor.ConfigList
 import nextflow.processor.ErrorStrategy
-import nextflow.script.params.*
+import nextflow.script.ProcessInputs
+import nextflow.script.ProcessOutputs
 import nextflow.script.BaseScript
 import nextflow.script.BodyDef
 import nextflow.script.ProcessConfig
 import nextflow.script.ProcessDef
 
 /**
- * Implements the process builder DSL.
+ * Builder for {@link ProcessDef}.
  *
  * @author Ben Sherman 
  */
@@ -414,130 +415,35 @@ class ProcessBuilder {
             allSecrets.add(name)
     }
 
-    /// INPUTS
-
-    InParam _in_each( Object obj ) {
-        new EachInParam(config).bind(obj)
-    }
-
-    InParam _in_env( Object obj ) {
-        new EnvInParam(config).bind(obj)
-    }
-
-    InParam _in_file( Object obj ) {
-        new FileInParam(config).bind(obj)
-    }
-
-    InParam _in_path( Map opts=[:], Object obj ) {
-        new FileInParam(config)
-                .setPathQualifier(true)
-                .setOptions(opts)
-                .bind(obj)
-    }
-
-    InParam _in_stdin( Object obj = null ) {
-        def result = new StdInParam(config)
-        if( obj )
-            result.bind(obj)
-        result
-    }
-
-    InParam _in_tuple( Object... obj ) {
-        if( obj.length < 2 )
-            throw new IllegalArgumentException("Input `tuple` must define at least two elements -- Check process `$processName`")
-        new TupleInParam(config).bind(obj)
-    }
-
-    InParam _in_val( Object obj ) {
-        new ValueInParam(config).bind(obj)
-    }
-
-    /// OUTPUTS
-
-    OutParam _out_env( Object obj ) {
-        new EnvOutParam(config)
-                .bind(obj)
-    }
-
-    OutParam _out_env( Map opts, Object obj ) {
-        new EnvOutParam(config)
-                .setOptions(opts)
-                .bind(obj)
-    }
-
-    OutParam _out_file( Object obj ) {
-        // note: check that is a String type to avoid to force
-        // the evaluation of GString object to a string
-        if( obj instanceof String && obj == '-' )
-            new StdOutParam(config).bind(obj)
-        else
-            new FileOutParam(config).bind(obj)
-    }
-
-    OutParam _out_path( Map opts=null, Object obj ) {
-        // note: check that is a String type to avoid to force
-        // the evaluation of GString object to a string
-        if( obj instanceof String && obj == '-' )
-            new StdOutParam(config)
-                    .setOptions(opts)
-                    .bind(obj)
-
-        else
-            new FileOutParam(config)
-                    .setPathQualifier(true)
-                    .setOptions(opts)
-                    .bind(obj)
-    }
-
-    OutParam _out_stdout( Map opts ) {
-        new StdOutParam(config)
-                .setOptions(opts)
-                .bind('-')
-    }
-
-    OutParam _out_stdout( obj = null ) {
-        def result = new StdOutParam(config).bind('-')
-        if( obj )
-            result.setInto(obj)
-        result
-    }
+    /// SCRIPT
 
-    OutParam _out_tuple( Object... obj ) {
-        if( obj.length < 2 )
-            throw new IllegalArgumentException("Output `tuple` must define at least two elements -- Check process `$processName`")
-        new TupleOutParam(config)
-                .bind(obj)
+    ProcessBuilder withInputs(ProcessInputs inputs) {
+        config.inputs = inputs
+        return this
     }
 
-    OutParam _out_tuple( Map opts, Object... obj ) {
-        if( obj.length < 2 )
-            throw new IllegalArgumentException("Output `tuple` must define at least two elements -- Check process `$processName`")
-        new TupleOutParam(config)
-                .setOptions(opts)
-                .bind(obj)
+    ProcessBuilder withOutputs(ProcessOutputs outputs) {
+        config.outputs = outputs
+        return this
     }
 
-    OutParam _out_val( Object obj ) {
-        new ValueOutParam(config)
-                .bind(obj)
+    ProcessBuilder withBody(Closure closure, String section, String source='', List values=null) {
+        withBody(new BodyDef(closure, source, section, values))
     }
 
-    OutParam _out_val( Map opts, Object obj ) {
-        new ValueOutParam(config)
-                .setOptions(opts)
-                .bind(obj)
+    ProcessBuilder withBody(BodyDef body) {
+        this.body = body
+        return this
     }
 
-    /// SCRIPT
-
-    ProcessBuilder withParams(String[] params) {
-        config.setParams(params)
-        return this
+    ProcessConfig getConfig() {
+        return config
     }
 
-    ProcessBuilder withBody(BodyDef body) {
-        this.body = body
-        return this
+    ProcessDef build() {
+        if ( !body )
+            throw new ScriptRuntimeException("Missing script in the specified process block -- make sure it terminates with the script string to be executed")
+        return new ProcessDef(ownerScript, processName, body, config)
     }
 
     /// CONFIG
@@ -735,16 +641,4 @@ class ProcessBuilder {
             config.put(name, value)
         }
     }
-
-    /// BUILD
-
-    ProcessConfig getConfig() {
-        return config
-    }
-
-    ProcessDef build() {
-        if ( !body )
-            throw new ScriptRuntimeException("Missing script in the specified process block -- make sure it terminates with the script string to be executed")
-        return new ProcessDef(ownerScript, processName, body, config)
-    }
 }
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessDsl.groovy b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessDsl.groovy
new file mode 100644
index 0000000000..ae7129e76c
--- /dev/null
+++ b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessDsl.groovy
@@ -0,0 +1,147 @@
+/*
+ * Copyright 2013-2023, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package nextflow.script.dsl
+
+import nextflow.script.BaseScript
+import nextflow.script.params.*
+
+/**
+ * Implements the process DSL.
+ *
+ * @author Paolo Di Tommaso 
+ */
+class ProcessDsl extends ProcessBuilder {
+
+    ProcessDsl(BaseScript ownerScript, String processName) {
+        super(ownerScript, processName)
+    }
+
+    /// INPUTS
+
+    InParam _in_each( Object obj ) {
+        new EachInParam(config).bind(obj)
+    }
+
+    InParam _in_env( Object obj ) {
+        new EnvInParam(config).bind(obj)
+    }
+
+    InParam _in_file( Object obj ) {
+        new FileInParam(config).bind(obj)
+    }
+
+    InParam _in_path( Map opts=[:], Object obj ) {
+        new FileInParam(config)
+                .setPathQualifier(true)
+                .setOptions(opts)
+                .bind(obj)
+    }
+
+    InParam _in_stdin( Object obj = null ) {
+        def result = new StdInParam(config)
+        if( obj )
+            result.bind(obj)
+        result
+    }
+
+    InParam _in_tuple( Object... obj ) {
+        if( obj.length < 2 )
+            throw new IllegalArgumentException("Input `tuple` must define at least two elements -- Check process `$processName`")
+        new TupleInParam(config).bind(obj)
+    }
+
+    InParam _in_val( Object obj ) {
+        new ValueInParam(config).bind(obj)
+    }
+
+    /// OUTPUTS
+
+    OutParam _out_env( Object obj ) {
+        new EnvOutParam(config)
+                .bind(obj)
+    }
+
+    OutParam _out_env( Map opts, Object obj ) {
+        new EnvOutParam(config)
+                .setOptions(opts)
+                .bind(obj)
+    }
+
+    OutParam _out_file( Object obj ) {
+        // note: check that is a String type to avoid to force
+        // the evaluation of GString object to a string
+        if( obj instanceof String && obj == '-' )
+            new StdOutParam(config).bind(obj)
+        else
+            new FileOutParam(config).bind(obj)
+    }
+
+    OutParam _out_path( Map opts=null, Object obj ) {
+        // note: check that is a String type to avoid to force
+        // the evaluation of GString object to a string
+        if( obj instanceof String && obj == '-' )
+            new StdOutParam(config)
+                    .setOptions(opts)
+                    .bind(obj)
+
+        else
+            new FileOutParam(config)
+                    .setPathQualifier(true)
+                    .setOptions(opts)
+                    .bind(obj)
+    }
+
+    OutParam _out_stdout( Map opts ) {
+        new StdOutParam(config)
+                .setOptions(opts)
+                .bind('-')
+    }
+
+    OutParam _out_stdout( obj = null ) {
+        def result = new StdOutParam(config).bind('-')
+        if( obj )
+            result.setInto(obj)
+        result
+    }
+
+    OutParam _out_tuple( Object... obj ) {
+        if( obj.length < 2 )
+            throw new IllegalArgumentException("Output `tuple` must define at least two elements -- Check process `$processName`")
+        new TupleOutParam(config)
+                .bind(obj)
+    }
+
+    OutParam _out_tuple( Map opts, Object... obj ) {
+        if( obj.length < 2 )
+            throw new IllegalArgumentException("Output `tuple` must define at least two elements -- Check process `$processName`")
+        new TupleOutParam(config)
+                .setOptions(opts)
+                .bind(obj)
+    }
+
+    OutParam _out_val( Object obj ) {
+        new ValueOutParam(config)
+                .bind(obj)
+    }
+
+    OutParam _out_val( Map opts, Object obj ) {
+        new ValueOutParam(config)
+                .setOptions(opts)
+                .bind(obj)
+    }
+
+}
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessInputsBuilder.groovy b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessInputsBuilder.groovy
index 22f05a0c43..227c932d47 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessInputsBuilder.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessInputsBuilder.groovy
@@ -17,46 +17,55 @@
 package nextflow.script.dsl
 
 import groovy.transform.CompileStatic
-import groovy.util.logging.Slf4j
-import nextflow.processor.TaskFileInput
-import nextflow.script.ProcessConfig
+import nextflow.script.ProcessFileInput
+import nextflow.script.ProcessInput
+import nextflow.script.ProcessInputs
 
 /**
- * Process inputs builder DSL for the {@code ProcessFn} annotation.
+ * Builder for {@link ProcessInputs}.
  *
  * @author Ben Sherman 
  */
-@Slf4j
 @CompileStatic
 class ProcessInputsBuilder {
 
-    private ProcessConfig config
+    private ProcessInputs inputs = new ProcessInputs()
 
-    ProcessInputsBuilder(ProcessConfig config) {
-        this.config = config
+    ProcessInputsBuilder env(String name, Object source) {
+        inputs.env.put(name, source)
+        return this
     }
 
-    void env( String name, Object obj ) {
-        final allEnvs = (Map)config.get('env', [:])
-        allEnvs.put(name, obj)
+    ProcessInputsBuilder path(Map opts=[:], Object source) {
+        inputs.files.add(new ProcessFileInput(source, null, true, opts))
+        return this
     }
 
-    void file( Object obj ) {
-        final allFiles = (List)config.get('files', [])
-        allFiles.add(new TaskFileInput(obj, false, null, [:]))
+    ProcessInputsBuilder stdin(Object source) {
+        inputs.stdin = source
+        return this
     }
 
-    void path( Map opts=[:], Object obj ) {
-        final allFiles = (List)config.get('files', [])
-        allFiles.add(new TaskFileInput(obj, true, null, opts))
+    /**
+     * Declare a process input parameter which will be
+     * bound when the task is created and can be referenced by
+     * other process input methods. For example:
+     *
+     *   take 'sample_id'
+     *   take 'files'
+     *
+     *   env('SAMPLE_ID') { sample_id }
+     *   path { files }
+     *
+     * @param name
+     */
+    ProcessInputsBuilder take(String name) {
+        inputs.add(new ProcessInput(name))
+        return this
     }
 
-    void stdin( Object obj ) {
-        config.put('stdin', obj)
-    }
-
-    ProcessConfig getConfig() {
-        return config
+    ProcessInputs build() {
+        return inputs
     }
 
 }
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessOutputsBuilder.groovy b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessOutputsBuilder.groovy
new file mode 100644
index 0000000000..52b1e72f64
--- /dev/null
+++ b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessOutputsBuilder.groovy
@@ -0,0 +1,76 @@
+/*
+ * Copyright 2013-2023, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package nextflow.script.dsl
+
+import groovy.transform.CompileStatic
+import nextflow.script.ProcessFileOutput
+import nextflow.script.ProcessOutput
+import nextflow.script.ProcessOutputs
+
+/**
+ * Builder for {@link ProcessOutputs}.
+ *
+ * @author Ben Sherman 
+ */
+@CompileStatic
+class ProcessOutputsBuilder {
+
+    private ProcessOutputs outputs = new ProcessOutputs()
+
+    ProcessOutputsBuilder env(String name) {
+        env(name, name)
+    }
+
+    ProcessOutputsBuilder env(String name, String target) {
+        outputs.env.put(name, target)
+        return this
+    }
+
+    ProcessOutputsBuilder path(Map opts=[:], String name) {
+        path(opts, name, name)
+    }
+
+    ProcessOutputsBuilder path(Map opts=[:], String name, Object target) {
+        outputs.files.put(name, new ProcessFileOutput(target, opts))
+        return this
+    }
+
+    /**
+     * Declare a process output with a closure that will
+     * be evaluated after the task execution. For example:
+     *
+     *   env 'SAMPLE_ID'           // declare output env 'SAMPLE_ID'
+     *   path '$out0', 'file.txt'  // declare output file 'file.txt'
+     *
+     *   emit { sample_id }        // variable 'sample_id' in task context
+     *   emit { stdout }           // standard output of task script
+     *   emit { [env('SAMPLE_ID'), path('$out0')] }
+     *   emit { new Sample(sample_id, path('$out0')) }
+     *
+     * @param opts
+     * @param target
+     */
+    ProcessOutputsBuilder emit(Map opts=[:], Object target) {
+        outputs.add(new ProcessOutput(outputs, target, opts))
+        return this
+    }
+
+    ProcessOutputs build() {
+        outputs
+    }
+
+}
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/DefaultOutParam.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/DefaultOutParam.groovy
deleted file mode 100644
index ab0995f9b2..0000000000
--- a/modules/nextflow/src/main/groovy/nextflow/script/params/DefaultOutParam.groovy
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Copyright 2013-2023, Seqera Labs
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package nextflow.script.params
-
-
-import groovyx.gpars.dataflow.DataflowQueue
-import nextflow.script.ProcessConfig
-/**
- * Model a process default output parameter
- *
- * @author Paolo Di Tommaso 
- */
-final class DefaultOutParam extends BaseOutParam {
-
-    static enum Completion { DONE }
-
-    DefaultOutParam(ProcessConfig config ) {
-        super(config)
-        bind('-')
-        setInto(new DataflowQueue())
-    }
-}
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/FileOutParam.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/FileOutParam.groovy
index 2a3f94bb6f..8b62db0c3f 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/params/FileOutParam.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/params/FileOutParam.groovy
@@ -24,7 +24,7 @@ import groovy.util.logging.Slf4j
 import nextflow.NF
 import nextflow.exception.IllegalFileException
 import nextflow.file.FilePatternSplitter
-import nextflow.processor.PathArityAware
+import nextflow.script.PathArityAware
 import nextflow.script.TokenVar
 import nextflow.util.BlankSeparatedList
 /**
@@ -159,14 +159,6 @@ class FileOutParam extends BaseOutParam implements OutParam, OptionalParam, Path
 
     @PackageScope String getFilePattern() { filePattern }
 
-    @PackageScope
-    static String clean(String path) {
-        while (path.startsWith('/') ) {
-            path = path.substring(1)
-        }
-        return path
-    }
-
     @PackageScope
     String relativize(String path, Path workDir) {
         if( !path.startsWith('/') )
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/InputsList.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/InputsList.groovy
deleted file mode 100644
index c389fe4e80..0000000000
--- a/modules/nextflow/src/main/groovy/nextflow/script/params/InputsList.groovy
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Copyright 2013-2023, Seqera Labs
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package nextflow.script.params
-
-import groovy.util.logging.Slf4j
-import groovyx.gpars.dataflow.DataflowReadChannel
-import nextflow.extension.CH
-
-
-/**
- * Container to hold all process outputs
- *
- * @author Paolo Di Tommaso 
- */
-@Slf4j
-class InputsList implements List, Cloneable {
-
-    @Override
-    InputsList clone() {
-        def result = (InputsList)super.clone()
-        result.target = new ArrayList<>(target.size())
-        for( InParam param : target ) {
-            result.target.add((InParam)param.clone())
-        }
-        return result
-    }
-
-    @Delegate
-    private List target = new LinkedList<>()
-
-    List getChannels() {
-        target.collect { InParam it -> it.getInChannel() }
-    }
-
-    List getNames() { target *. name }
-
-
-    def  List ofType( Class clazz ) {
-        (List) target.findAll { it.class == clazz }
-    }
-
-    boolean allScalarInputs() {
-        for( InParam param : target ) {
-            if( CH.isChannelQueue(param.inChannel) )
-                return false
-        }
-        return true
-    }
-
-}
-
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/MissingParam.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/MissingParam.groovy
deleted file mode 100644
index b8385c27b9..0000000000
--- a/modules/nextflow/src/main/groovy/nextflow/script/params/MissingParam.groovy
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Copyright 2013-2023, Seqera Labs
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package nextflow.script.params
-
-
-/**
- * Placeholder trait to mark a missing optional output parameter
- *
- * @author Paolo Di Tommaso 
- */
-trait MissingParam {
-
-    OutParam missing
-
-}
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/OutputsList.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/OutputsList.groovy
deleted file mode 100644
index 9509c1ecec..0000000000
--- a/modules/nextflow/src/main/groovy/nextflow/script/params/OutputsList.groovy
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Copyright 2013-2023, Seqera Labs
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package nextflow.script.params
-
-import groovyx.gpars.dataflow.DataflowWriteChannel
-
-
-/**
- * Container to hold all process outputs
- *
- * @author Paolo Di Tommaso 
- */
-class OutputsList implements List, Cloneable {
-
-    @Override
-    OutputsList clone() {
-        def result = (OutputsList)super.clone()
-        result.target = new ArrayList<>(target.size())
-        for( OutParam param : target )
-            result.add((OutParam)param.clone())
-        return result
-    }
-
-    @Delegate
-    private List target = new LinkedList<>()
-
-    List getChannels() {
-        final List result = new ArrayList<>(target.size())
-        for(OutParam param : target) { result.add(param.getOutChannel()) }
-        return result
-    }
-
-    List getNames() { target *. name }
-
-    def  List ofType( Class... classes ) {
-        (List) target.findAll { it.class in classes }
-    }
-
-    void setSingleton( boolean value ) {
-        for( OutParam param : target ) { ((BaseOutParam)param).singleton = value }
-    }
-}

From d2268b2d7aeb4fc1ef884e13d0895ecfdd04412c Mon Sep 17 00:00:00 2001
From: Ben Sherman 
Date: Fri, 8 Dec 2023 22:13:29 -0600
Subject: [PATCH 14/36] Refactor process inputs/outputs DSL

Signed-off-by: Ben Sherman 
---
 .../groovy/nextflow/ast/DslCodeVistor.groovy  |  14 +-
 .../main/groovy/nextflow/cache/CacheDB.groovy |   5 +-
 .../nextflow/processor/TaskConfig.groovy      | 207 +----------
 .../processor/TaskFileCollector.groovy        |   2 +-
 .../processor/TaskOutputCollector.groovy      | 122 +++++++
 .../nextflow/processor/TaskProcessor.groovy   |  91 ++---
 .../groovy/nextflow/processor/TaskRun.groovy  |  74 ++--
 .../groovy/nextflow/script/IncludeDef.groovy  |   9 +-
 .../nextflow/script/ProcessConfig.groovy      |   2 -
 .../groovy/nextflow/script/ProcessDef.groovy  |   2 +-
 .../nextflow/script/ProcessFileInput.groovy   |  32 +-
 .../nextflow/script/ProcessFileOutput.groovy  |  33 +-
 .../nextflow/script/ProcessInput.groovy       |  36 +-
 .../nextflow/script/ProcessInputs.groovy      |  46 ++-
 .../nextflow/script/ProcessOutput.groovy      | 115 +-----
 .../nextflow/script/ProcessOutputs.groovy     |  43 ++-
 .../nextflow/script/ScriptTokens.groovy       |  26 +-
 .../nextflow/script/dsl/ProcessBuilder.groovy |  23 +-
 .../nextflow/script/dsl/ProcessDsl.groovy     | 331 ++++++++++++++----
 .../script/dsl/ProcessInputsBuilder.groovy    |   4 +-
 .../script/dsl/ProcessOutputsBuilder.groovy   |  16 +-
 .../nextflow/script/params/BaseInParam.groovy | 200 -----------
 .../script/params/BaseOutParam.groovy         | 185 ----------
 .../nextflow/script/params/BaseParam.groovy   | 170 ---------
 .../nextflow/script/params/EachInParam.groovy |  93 -----
 .../nextflow/script/params/EnvInParam.groovy  |  33 --
 .../nextflow/script/params/EnvOutParam.groovy |  48 ---
 .../nextflow/script/params/FileInParam.groovy |  78 -----
 .../script/params/FileOutParam.groovy         | 215 ------------
 .../nextflow/script/params/InParam.groovy     |  40 ---
 .../script/params/OptionalParam.groovy        |  36 --
 .../nextflow/script/params/OutParam.groovy    |  43 ---
 .../script/params/PathQualifier.groovy        |  30 --
 .../nextflow/script/params/StdInParam.groovy  |  38 --
 .../nextflow/script/params/StdOutParam.groovy |  27 --
 .../script/params/TupleInParam.groovy         | 103 ------
 .../script/params/TupleOutParam.groovy        | 101 ------
 .../script/params/ValueInParam.groovy         |  33 --
 .../script/params/ValueOutParam.groovy        |  73 ----
 .../groovy/nextflow/util/LazyHelper.groovy    | 285 +++++++++++++++
 40 files changed, 912 insertions(+), 2152 deletions(-)
 create mode 100644 modules/nextflow/src/main/groovy/nextflow/processor/TaskOutputCollector.groovy
 delete mode 100644 modules/nextflow/src/main/groovy/nextflow/script/params/BaseInParam.groovy
 delete mode 100644 modules/nextflow/src/main/groovy/nextflow/script/params/BaseOutParam.groovy
 delete mode 100644 modules/nextflow/src/main/groovy/nextflow/script/params/BaseParam.groovy
 delete mode 100644 modules/nextflow/src/main/groovy/nextflow/script/params/EachInParam.groovy
 delete mode 100644 modules/nextflow/src/main/groovy/nextflow/script/params/EnvInParam.groovy
 delete mode 100644 modules/nextflow/src/main/groovy/nextflow/script/params/EnvOutParam.groovy
 delete mode 100644 modules/nextflow/src/main/groovy/nextflow/script/params/FileInParam.groovy
 delete mode 100644 modules/nextflow/src/main/groovy/nextflow/script/params/FileOutParam.groovy
 delete mode 100644 modules/nextflow/src/main/groovy/nextflow/script/params/InParam.groovy
 delete mode 100644 modules/nextflow/src/main/groovy/nextflow/script/params/OptionalParam.groovy
 delete mode 100644 modules/nextflow/src/main/groovy/nextflow/script/params/OutParam.groovy
 delete mode 100644 modules/nextflow/src/main/groovy/nextflow/script/params/PathQualifier.groovy
 delete mode 100644 modules/nextflow/src/main/groovy/nextflow/script/params/StdInParam.groovy
 delete mode 100644 modules/nextflow/src/main/groovy/nextflow/script/params/StdOutParam.groovy
 delete mode 100644 modules/nextflow/src/main/groovy/nextflow/script/params/TupleInParam.groovy
 delete mode 100644 modules/nextflow/src/main/groovy/nextflow/script/params/TupleOutParam.groovy
 delete mode 100644 modules/nextflow/src/main/groovy/nextflow/script/params/ValueInParam.groovy
 delete mode 100644 modules/nextflow/src/main/groovy/nextflow/script/params/ValueOutParam.groovy
 create mode 100644 modules/nextflow/src/main/groovy/nextflow/util/LazyHelper.groovy

diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/DslCodeVistor.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/DslCodeVistor.groovy
index 9dffe5ddbf..d3d667c443 100644
--- a/modules/nextflow/src/main/groovy/nextflow/ast/DslCodeVistor.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/ast/DslCodeVistor.groovy
@@ -34,7 +34,7 @@ import nextflow.script.TokenStdinCall
 import nextflow.script.TokenStdoutCall
 import nextflow.script.TokenValCall
 import nextflow.script.TokenValRef
-import nextflow.script.TokenVar
+import nextflow.util.LazyVar
 import org.codehaus.groovy.ast.ASTNode
 import org.codehaus.groovy.ast.ClassCodeVisitorSupport
 import org.codehaus.groovy.ast.MethodNode
@@ -189,8 +189,8 @@ class DslCodeVisitor extends ClassCodeVisitorSupport {
             else if( arg instanceof VariableExpression ) {
                 // the name of the component i.e. process, workflow, etc to import
                 final component = arg.getName()
-                // wrap the name in a `TokenVar` type
-                final token = createX(TokenVar, new ConstantExpression(component))
+                // wrap the name in a `LazyVar` type
+                final token = createX(LazyVar, new ConstantExpression(component))
                 // create a new `IncludeDef` object
                 newArgs.addExpression(createX(IncludeDef, token))
             }
@@ -198,8 +198,8 @@ class DslCodeVisitor extends ClassCodeVisitorSupport {
                 def cast = (CastExpression)arg
                 // the name of the component i.e. process, workflow, etc to import
                 final component = (cast.expression as VariableExpression).getName()
-                // wrap the name in a `TokenVar` type
-                final token = createX(TokenVar, new ConstantExpression(component))
+                // wrap the name in a `LazyVar` type
+                final token = createX(LazyVar, new ConstantExpression(component))
                 // the alias to give it
                 final alias = constX(cast.type.name)
                 newArgs.addExpression( createX(IncludeDef, token, alias) )
@@ -1020,7 +1020,7 @@ class DslCodeVisitor extends ClassCodeVisitorSupport {
     protected Expression varToStrX( Expression expr ) {
         if( expr instanceof VariableExpression ) {
             def name = ((VariableExpression) expr).getName()
-            return createX( TokenVar, new ConstantExpression(name) )
+            return createX( LazyVar, new ConstantExpression(name) )
         }
         else if( expr instanceof PropertyExpression ) {
             // transform an output declaration such
@@ -1067,7 +1067,7 @@ class DslCodeVisitor extends ClassCodeVisitorSupport {
                 return createX( TokenStdoutCall )
 
             else
-                return createX( TokenVar, new ConstantExpression(name) )
+                return createX( LazyVar, new ConstantExpression(name) )
         }
 
         if( expr instanceof MethodCallExpression ) {
diff --git a/modules/nextflow/src/main/groovy/nextflow/cache/CacheDB.groovy b/modules/nextflow/src/main/groovy/nextflow/cache/CacheDB.groovy
index 214e3235e3..0238af186b 100644
--- a/modules/nextflow/src/main/groovy/nextflow/cache/CacheDB.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/cache/CacheDB.groovy
@@ -139,9 +139,8 @@ class CacheDB implements Closeable {
         final proc = task.processor
         final key = task.hash
 
-        // save the context map for caching purpose
-        // only the 'cache' is active and
-        TaskContext ctx = proc.isCacheable() && task.hasCacheableValues() ? task.context : null
+        // -- save the task context only if caching is enabled for the process
+        TaskContext ctx = proc.isCacheable() ? task.context : null
 
         def record = new ArrayList(3)
         record[0] = trace.serialize()
diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskConfig.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskConfig.groovy
index 512b76ed0e..05fd32a52f 100644
--- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskConfig.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskConfig.groovy
@@ -34,6 +34,7 @@ import nextflow.script.TaskClosure
 import nextflow.util.CmdLineHelper
 import nextflow.util.CmdLineOptionMap
 import nextflow.util.Duration
+import nextflow.util.LazyMap
 import nextflow.util.MemoryUnit
 /**
  * Task local configuration properties
@@ -45,9 +46,6 @@ class TaskConfig extends LazyMap implements Cloneable {
 
     static public final int EXIT_ZERO = 0
 
-    @PackageScope
-    static final List LAZY_MAP_PROPERTIES = ['env', 'ext']
-
     private transient Map cache = new LinkedHashMap(20)
 
     TaskConfig() {  }
@@ -81,10 +79,9 @@ class TaskConfig extends LazyMap implements Cloneable {
         // clear cache to force re-compute dynamic entries
         this.cache.clear()
 
-        // set the binding context for lazy map properties
-        for( def key in LAZY_MAP_PROPERTIES )
-            if( target.get(key) instanceof LazyMap )
-                (target.get(key) as LazyMap).binding = context
+        // set the binding context for 'ext' map
+        if( target.ext instanceof LazyMap )
+            (target.ext as LazyMap).binding = context
 
         // set the this object in the task context in order to allow task properties to be resolved in process script
         context.put(TASK_CONTEXT_PROPERTY_NAME, this)
@@ -144,7 +141,7 @@ class TaskConfig extends LazyMap implements Cloneable {
             return cache.get(key)
 
         def result
-        if( key in LAZY_MAP_PROPERTIES ) {
+        if( key == 'ext' ) {
             if( target.containsKey(key) )
                 result = target.get(key)
             else {
@@ -172,7 +169,7 @@ class TaskConfig extends LazyMap implements Cloneable {
             }
             target.put(key, value)
         }
-        else if( key in LAZY_MAP_PROPERTIES && value instanceof Map ) {
+        else if( key == 'ext' && value instanceof Map ) {
             super.put( key, new LazyMap(value) )
         }
         else {
@@ -522,195 +519,3 @@ class TaskConfig extends LazyMap implements Cloneable {
     }
 
 }
-
-/**
- * A map that resolve closure and gstring in a lazy manner
- */
-@CompileStatic
-class LazyMap implements Map {
-
-    /** The target map holding the values */
-    @Delegate
-    private Map target
-
-    /** The context map against which dynamic properties are resolved */
-    private Map binding
-
-    private boolean dynamic
-
-    protected boolean isDynamic() { dynamic }
-
-    protected void setDynamic(boolean val) { dynamic = val }
-
-    protected Map getBinding() { binding }
-
-    protected void setBinding(Map map) { this.binding = map }
-
-    protected Map getTarget() { target }
-
-    protected void setTarget(Map obj) { this.target = obj }
-
-    LazyMap() {
-        target = new HashMap<>()
-    }
-
-    LazyMap( Map entries ) {
-        assert entries != null
-        target = new HashMap<>()
-        putAll(entries)
-    }
-
-    /**
-     * Resolve a directive *dynamic* value i.e. defined with a closure or lazy string
-     *
-     * @param name The directive name
-     * @param value The value to be resolved
-     * @return The resolved value
-     */
-    protected resolve( String name, value ) {
-
-        /*
-         * directive with one value and optional named parameter are converted
-         * to a list object in which the first element is a map holding the named parameters
-         * and the second is the directive value
-         */
-        if( value instanceof ConfigList ) {
-            def copy = new ArrayList(value.size())
-            for( Object item : value ) {
-                if( item instanceof Map )
-                    copy.add( resolveParams(name, item as Map) )
-                else
-                    copy.add( resolveImpl(name, item) )
-            }
-            return copy
-        }
-
-        /*
-         * resolve the values in a map object
-         * note: 'ext' property is meant for extension attributes
-         * as it should be preserved as LazyMap
-         */
-        else if( value instanceof Map && name !in TaskConfig.LAZY_MAP_PROPERTIES ) {
-            return resolveParams(name, value)
-        }
-
-        /*
-         * simple value
-         */
-        else {
-            return resolveImpl(name, value)
-        }
-
-    }
-
-    /**
-     * Resolve directive *dynamic* named params
-     *
-     * @param name The directive name
-     * @param value The map holding the named params
-     * @return A map in which dynamic params are resolved to the actual value
-     */
-    private resolveParams( String name, Map value ) {
-
-        final copy = new LinkedHashMap()
-        final attr = (value as Map)
-        for( Entry entry : attr.entrySet() ) {
-            copy[entry.key] = resolveImpl(name, entry.value, true)
-        }
-        return copy
-    }
-
-    /**
-     * Resolve a directive dynamic value
-     *
-     * @param name The directive name
-     * @param value The value to be resolved
-     * @param param When {@code true} points that it is a named parameter value, thus closure are only cloned
-     * @return The resolved directive value
-     */
-    private resolveImpl( String name, value, boolean param=false ) {
-
-        if( value instanceof Closure ) {
-            def copy = value.cloneWith(getBinding())
-            if( param ) {
-                return copy
-            }
-
-            try {
-                return copy.call()
-            }
-            catch( MissingPropertyException e ) {
-                if( getBinding() == null ) throw new IllegalStateException("Directive `$name` doesn't support dynamic value (or context not yet initialized)")
-                else throw e
-            }
-        }
-
-        else if( value instanceof GString ) {
-            return value.cloneAsLazy(getBinding()).toString()
-        }
-
-        return value
-    }
-
-    /**
-     * Override the get method in such a way that {@link Closure} values are resolved against
-     * the {@link #binding} map
-     *
-     * @param key The map entry key
-     * @return The associated value
-     */
-    Object get( key ) {
-        return getValue(key)
-    }
-
-    Object getValue(Object key) {
-        final value = target.get(key)
-        return resolve(key as String, value)
-    }
-
-    Object put( String key, Object value ) {
-        if( value instanceof Closure ) {
-            dynamic |= true
-        }
-        else if( value instanceof GString ) {
-            for( int i=0; i put(k as String, v) }
-    }
-
-    @Override
-    String toString() {
-        final allKeys = keySet()
-        final result = new ArrayList(allKeys.size())
-        for( String key : allKeys ) { result << "$key: ${getProperty(key)}".toString() }
-        result.join('; ')
-    }
-
-}
-
-@CompileStatic
-class ConfigList implements List {
-
-    @Delegate
-    private List target
-
-    ConfigList() {
-        target = []
-    }
-
-    ConfigList(int size) {
-        target = new ArrayList(size)
-    }
-
-    ConfigList(Collection items) {
-        target = new ArrayList(items)
-    }
-
-}
diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskFileCollector.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskFileCollector.groovy
index 2219cc4c1c..b0bacfdceb 100644
--- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskFileCollector.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskFileCollector.groovy
@@ -127,7 +127,7 @@ class TaskFileCollecter {
      */
     protected List excludeStagedInputs(TaskRun task, List collectedFiles) {
 
-        final List allStagedFiles = task.getStagedInputs()
+        final List allStagedFiles = task.inputFiles.collect { it.stageName }
         final List result = new ArrayList<>(collectedFiles.size())
 
         for( int i = 0; i < collectedFiles.size(); i++ ) {
diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskOutputCollector.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskOutputCollector.groovy
new file mode 100644
index 0000000000..fe6ddbdc7c
--- /dev/null
+++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskOutputCollector.groovy
@@ -0,0 +1,122 @@
+/*
+ * Copyright 2013-2023, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package nextflow.processor
+
+import java.nio.file.Path
+
+import groovy.transform.CompileStatic
+import groovy.transform.Memoized
+import groovy.util.logging.Slf4j
+import nextflow.exception.MissingFileException
+import nextflow.exception.MissingValueException
+import nextflow.script.ProcessOutputs
+import nextflow.script.ScriptType
+/**
+ * Implements the resolution of task outputs
+ *
+ * @author Ben Sherman 
+ */
+@Slf4j
+@CompileStatic
+class TaskOutputCollector implements Map {
+
+    private ProcessOutputs declaredOutputs
+
+    private boolean optional
+
+    private TaskRun task
+
+    @Delegate
+    private Map delegate
+
+    TaskOutputCollector(ProcessOutputs declaredOutputs, boolean optional, TaskRun task) {
+        this.declaredOutputs = declaredOutputs
+        this.optional = optional
+        this.task = task
+        this.delegate = task.context
+    }
+
+    /**
+     * Get an environment variable from the task environment.
+     *
+     * @param key
+     */
+    String env(String key) {
+        final varName = declaredOutputs.getEnv().get(key)
+        final result = env0(task.workDir).get(varName)
+
+        if( result == null && !optional )
+            throw new MissingValueException("Missing environment variable: $varName")
+
+        return result
+    }
+
+    @Memoized
+    static private Map env0(Path workDir) {
+        new TaskEnvCollector(workDir).collect()
+    }
+
+    /**
+     * Get a file or list of files from the task environment.
+     *
+     * @param key
+     */
+    Object path(String key) {
+        final param = declaredOutputs.getFiles().get(key)
+        final result = new TaskFileCollecter(param, task).collect()
+
+        if( result instanceof Path )
+            task.outputFiles.add(result)
+        else if( result instanceof Collection )
+            task.outputFiles.addAll(result)
+
+        return result
+    }
+
+    /**
+     * Get the standard output from the task environment.
+     */
+    Object stdout() {
+        final result = task.@stdout
+
+        if( result == null && task.type == ScriptType.SCRIPTLET )
+            throw new IllegalArgumentException("Missing 'stdout' for process > ${task.lazyName()}")
+
+        if( result instanceof Path && !result.exists() )
+            throw new MissingFileException("Missing 'stdout' file: ${result.toUriString()} for process > ${task.lazyName()}")
+
+        return result
+    }
+
+    /**
+     * Get a variable from the task context.
+     *
+     * @param name
+     */
+    @Override
+    Object get(Object name) {
+        if( name == 'stdout' )
+            return stdout()
+
+        try {
+            return delegate.get(name)
+        }
+        catch( MissingPropertyException e ) {
+            throw new MissingValueException("Missing variable in process output: ${e.property}")
+        }
+    }
+}
diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy
index 2a8a9622a7..a491da6f6a 100644
--- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy
@@ -81,12 +81,11 @@ import nextflow.script.ScriptMeta
 import nextflow.script.ScriptType
 import nextflow.script.TaskClosure
 import nextflow.script.bundle.ResourcesBundle
-import nextflow.script.params.FileOutParam
-import nextflow.script.params.ValueOutParam
 import nextflow.util.ArrayBag
 import nextflow.util.BlankSeparatedList
 import nextflow.util.CacheHelper
 import nextflow.util.Escape
+import nextflow.util.LazyHelper
 import nextflow.util.LockManager
 import nextflow.util.LoggerHelper
 import nextflow.util.TestOnly
@@ -458,8 +457,7 @@ class TaskProcessor {
         currentTask.set(task)
 
         // -- map the inputs to a map and use to delegate closure values interpolation
-        makeTaskContextStage1(task, values)
-        makeTaskContextStage2(task)
+        resolveTaskInputs(task, values)
 
         // verify that `when` guard, when specified, is satisfied
         if( !checkWhenGuard(task) )
@@ -645,22 +643,6 @@ class TaskProcessor {
             return false
         }
 
-        // -- when store path is set, only output params of type 'file' can be specified
-        final ctx = task.context
-        def invalid = task.getOutputs().keySet().any {
-            if( it instanceof ValueOutParam ) {
-                return !ctx.containsKey(it.name)
-            }
-            if( it instanceof FileOutParam ) {
-                return false
-            }
-            return true
-        }
-        if( invalid ) {
-            checkWarn "[${safeTaskName(task)}] StoreDir can only be used when using 'file' outputs"
-            return false
-        }
-
         if( !task.config.getStoreDir().exists() ) {
             log.trace "[${safeTaskName(task)}] Store dir does not exists > ${task.config.storeDir} -- return false"
             // no folder -> no cached result
@@ -728,7 +710,7 @@ class TaskProcessor {
             return false
         }
 
-        if( task.hasCacheableValues() && !entry.context ) {
+        if( !entry.context ) {
             log.trace "[${safeTaskName(task)}] Missing cache context -- return false"
             return false
         }
@@ -1507,34 +1489,33 @@ class TaskProcessor {
         return script.join('\n')
     }
 
-    final protected void makeTaskContextStage1( TaskRun task, List values ) {
+    final protected void resolveTaskInputs( TaskRun task, List values ) {
 
         final inputs = config.getInputs()
+        final ctx = task.context
 
-        // -- add variables
-        final vars = [:]
+        // -- add input params to task context
         for( int i = 0; i < inputs.size(); i++ )
-            vars.put(inputs[i].getName(), values[i])
+            ctx.put(inputs[i].getName(), values[i])
 
-        task.config.put('vars', vars)
-        task.context.putAll(vars)
+        // -- resolve local variables
+        for( def entry : inputs.getVariables() )
+            ctx.put(entry.key, LazyHelper.resolve(ctx, entry.value))
 
-        // -- add environment vars, stdin
-        task.config.put('env', new LazyMap(inputs.env))
-        task.config.put('stdin', inputs.stdin)
-    }
+        // -- resolve environment vars
+        for( def entry : inputs.getEnv() )
+            task.env.put(entry.key, LazyHelper.resolve(ctx, entry.value))
 
-    final protected void makeTaskContextStage2( TaskRun task ) {
+        // -- resolve stdin
+        task.stdin = LazyHelper.resolve(ctx, inputs.stdin)
 
-        final ctx = task.context
+        // -- resolve input files
         final allNames = new HashMap()
         int count = 0
+        final batch = session.filePorter.newBatch(executor.getStageDir())
 
-        final FilePorter.Batch batch = session.filePorter.newBatch(executor.getStageDir())
-
-        // -- resolve input files against the task context
-        for( def param : config.getInputs().files ) {
-            final val = param.getValue(ctx)
+        for( def param : config.getInputs().getFiles() ) {
+            final val = param.resolve(ctx)
             final normalized = normalizeInputToFiles(val, count, param.isPathQualifier(), batch)
             final resolved = expandWildcards( param.getFilePattern(ctx), normalized )
 
@@ -1571,17 +1552,6 @@ class TaskProcessor {
         session.filePorter.transfer(batch)
     }
 
-    final protected void makeTaskContextStage3( TaskRun task, HashCode hash, Path folder ) {
-
-        // set hash-code & working directory
-        task.hash = hash
-        task.workDir = folder
-        task.config.workDir = folder
-        task.config.hash = hash.toString()
-        task.config.name = task.getName()
-
-    }
-
     final protected HashCode createTaskHashKey(TaskRun task) {
 
         List keys = [ session.uniqueId, name, task.source ]
@@ -1590,10 +1560,18 @@ class TaskProcessor {
             keys << task.getContainerFingerprint()
 
         // add task inputs
-        keys.add( task.config.get('vars') )
-        keys.add( task.inputFiles )
-        keys.add( task.getInputEnvironment() )
-        keys.add( task.stdin )
+        final inputs = config.getInputs()
+        final inputVars = inputs.getNames() - inputs.getFiles()*.getName()
+        for( String var : inputVars ) {
+            keys.add(var)
+            keys.add(task.context.get(var))
+        }
+        if( task.env )
+            keys.add(task.env)
+        if( task.inputFiles )
+            keys.add(task.inputFiles)
+        if( task.stdin )
+            keys.add(task.stdin)
 
         // add all variable references in the task script but not declared as input/output
         def vars = getTaskGlobalVars(task)
@@ -1724,7 +1702,12 @@ class TaskProcessor {
     final protected void submitTask( TaskRun task, HashCode hash, Path folder ) {
         log.trace "[${safeTaskName(task)}] actual run folder: ${folder}"
 
-        makeTaskContextStage3(task, hash, folder)
+        // set hash-code & working directory
+        task.hash = hash
+        task.workDir = folder
+        task.config.workDir = folder
+        task.config.hash = hash.toString()
+        task.config.name = task.getName()
 
         // add the task to the collection of running tasks
         executor.submit(task)
diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy
index 020e070252..6fef194101 100644
--- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy
@@ -39,10 +39,8 @@ import nextflow.script.BodyDef
 import nextflow.script.ScriptType
 import nextflow.script.TaskClosure
 import nextflow.script.bundle.ResourcesBundle
-import nextflow.script.params.FileOutParam
-import nextflow.script.params.OutParam
-import nextflow.script.params.ValueOutParam
 import nextflow.spack.SpackCache
+import nextflow.util.ArrayBag
 import org.codehaus.groovy.runtime.MethodClosure
 /**
  * Models a task instance
@@ -81,16 +79,25 @@ class TaskRun implements Cloneable {
     TaskProcessor processor
 
     /**
-     * The list of resolved input files
+     * The map of input environment vars
+     *
+     * @see TaskProcessor#resolveTaskInputs(TaskRun, List)
      */
-    List inputFiles = []
+    Map env = [:]
 
     /**
-     * The list of resolved output files
+     * The list of input files
      *
-     * @see ProcessOutput#path(String)
+     * @see TaskProcessor#resolveTaskInputs(TaskRun, List)
      */
-    Set outputFiles = []
+    List inputFiles = new ArrayBag()
+
+    /**
+     * The value to be piped to the process stdin
+     *
+     * @see TaskProcessor#resolveTaskInputs(TaskRun, List)
+     */
+    def stdin
 
     /**
      * The list of resolved task outputs
@@ -99,13 +106,12 @@ class TaskRun implements Cloneable {
      */
     List outputs = []
 
-
     /**
-     * @return The value to be piped to the process stdin
+     * The list of resolved output files
+     *
+     * @see ProcessOutput.ResolverContext#path(String)
      */
-    def getStdin() {
-        config.get('stdin')
-    }
+    Set outputFiles = []
 
     /**
      * The exit code returned by executing the task script
@@ -387,30 +393,6 @@ class TaskRun implements Cloneable {
             : getScript()
     }
 
-
-    /**
-     * Check whenever there are values to be cached
-     */
-    boolean hasCacheableValues() {
-
-        if( config?.isDynamic() )
-            return true
-
-        for( OutParam it : outputs.keySet() ) {
-            if( it.class == ValueOutParam ) return true
-            if( it.class == FileOutParam && ((FileOutParam)it).isDynamic() ) return true
-        }
-
-        return false
-    }
-
-    /**
-     * Return the list of all input files staged as inputs by this task execution
-     */
-    List getStagedInputs()  {
-        inputFiles.collect { it.stageName }
-    }
-
     /**
      * @return A map object containing all the task input files as  pairs
      */
@@ -433,17 +415,6 @@ class TaskRun implements Cloneable {
         return result.unique()
     }
 
-    /**
-     * @return A map containing the task environment defined as input declaration by this task
-     */
-    protected Map getInputEnvironment() {
-        final Map environment = [:]
-        final allEnvs = config.get('env')
-        for( def key : allEnvs.keySet() )
-            environment.put(key, allEnvs.get(key))
-        return environment
-    }
-
     /**
      * @return A map representing the task execution environment
      */
@@ -453,7 +424,7 @@ class TaskRun implements Cloneable {
         // IMPORTANT: when copying the environment map a LinkedHashMap must be used to preserve
         // the insertion order of the env entries (ie. export FOO=1; export BAR=$FOO)
         final result = new LinkedHashMap( getProcessor().getProcessEnvironment() )
-        result.putAll( getInputEnvironment() )
+        result.putAll( env )
         return result
     }
 
@@ -525,7 +496,7 @@ class TaskRun implements Cloneable {
     }
 
     List getOutputEnvNames() {
-        final declaredOutputs = processor.getConfig().getOutputs()
+        final declaredOutputs = processor.config.getOutputs()
         return new ArrayList(declaredOutputs.env.values())
     }
 
@@ -657,11 +628,12 @@ class TaskRun implements Cloneable {
         // -- initialize the task code to be executed
         this.code = body.closure
 
-        // -- provide arguments directly or via delegate
         if( code instanceof MethodClosure ) {
+            // -- invoke task closure with arguments
             code = code.curry(args)
         }
         else {
+            // -- invoke task closure with delegate
             code = code.clone() as Closure
             code.setDelegate(this.context)
             code.setResolveStrategy(Closure.DELEGATE_ONLY)
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/IncludeDef.groovy b/modules/nextflow/src/main/groovy/nextflow/script/IncludeDef.groovy
index 3dd0ea1aad..769ae9fb4d 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/IncludeDef.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/IncludeDef.groovy
@@ -32,6 +32,7 @@ import groovy.util.logging.Slf4j
 import nextflow.NF
 import nextflow.Session
 import nextflow.exception.IllegalModulePath
+import nextflow.util.LazyVar
 /**
  * Implements a script inclusion
  *
@@ -56,15 +57,11 @@ class IncludeDef {
     @PackageScope Map addedParams
     private Session session
 
-    IncludeDef(TokenVar token, String alias=null) {
+    IncludeDef(LazyVar token, String alias=null) {
         def component = token.name; if(alias) component += " as $alias"
         def msg = "Unwrapped module inclusion is deprecated -- Replace `include $component from './MODULE/PATH'` with `include { $component } from './MODULE/PATH'`"
-        if( NF.isDsl2() )
-            throw new DeprecationException(msg)
-        log.warn msg
 
-        this.modules = new ArrayList<>(1)
-        this.modules << new Module(token.name, alias)
+        throw new DeprecationException(msg)
     }
 
     protected IncludeDef(List modules) {
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessConfig.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessConfig.groovy
index e6b8fbeae1..b85a135856 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessConfig.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessConfig.groovy
@@ -110,13 +110,11 @@ class ProcessConfig implements Map, Cloneable {
         return this
     }
 
-    @PackageScope
     ProcessConfig setInputs(ProcessInputs inputs) {
         this.inputs = inputs
         return this
     }
 
-    @PackageScope
     ProcessConfig setOutputs(ProcessOutputs outputs) {
         this.outputs = outputs
         return this
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy
index b2162267cd..e74fc2dd40 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy
@@ -191,7 +191,7 @@ class ProcessDef extends BindableDef implements IterableDef, ChainableDef {
         // make sure no more than one queue channel is provided
         int count = 0
         for( int i = 0; i < inputs.size(); i++ )
-            if( CH.isChannelQueue(inputs[i]) )
+            if( CH.isChannelQueue(inputs[i]) && !declaredInputs[i].isIterator() )
                 count += 1
 
         if( count > 1 )
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessFileInput.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessFileInput.groovy
index 2f6488a64e..d468a96ab2 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessFileInput.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessFileInput.groovy
@@ -17,6 +17,7 @@
 package nextflow.script
 
 import groovy.transform.CompileStatic
+import nextflow.util.LazyHelper
 
 /**
  * Models a process file input, which defines a file
@@ -31,14 +32,17 @@ class ProcessFileInput implements PathArityAware {
 
     private String name
 
-    private boolean coerceToPath
+    /**
+     * Flag to support legacy `file` input.
+     */
+    private boolean pathQualifier
 
     private Object filePattern
 
-    ProcessFileInput(Object value, String name, boolean coerceToPath, Map opts) {
+    ProcessFileInput(Object value, String name, boolean pathQualifier, Map opts) {
         this.value = value
         this.name = name
-        this.coerceToPath = coerceToPath
+        this.pathQualifier = pathQualifier
         this.filePattern = opts.stageAs ?: opts.name
 
         for( Map.Entry entry : opts )
@@ -49,8 +53,8 @@ class ProcessFileInput implements PathArityAware {
         this.filePattern = value
     }
 
-    Object getValue(Map ctx) {
-        return resolve(ctx, value)
+    Object resolve(Map ctx) {
+        return LazyHelper.resolve(ctx, value)
     }
 
     String getName() {
@@ -58,24 +62,14 @@ class ProcessFileInput implements PathArityAware {
     }
 
     boolean isPathQualifier() {
-        return coerceToPath
+        return pathQualifier
     }
 
     String getFilePattern(Map ctx) {
         if( filePattern != null )
-            return resolve(ctx, filePattern)
-
-        return filePattern = '*'
-    }
-
-    protected Object resolve(Map ctx, Object value) {
-        if( value instanceof GString )
-            return value.cloneAsLazy(ctx)
-
-        if( value instanceof Closure )
-            return ctx.with(value)
-
-        return value
+            return LazyHelper.resolve(ctx, filePattern)
+        else
+            return filePattern = '*'
     }
 
 }
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessFileOutput.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessFileOutput.groovy
index 92e4c15f81..9c5e43ccaa 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessFileOutput.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessFileOutput.groovy
@@ -23,6 +23,7 @@ import groovy.util.logging.Slf4j
 import nextflow.exception.IllegalFileException
 import nextflow.file.FilePatternSplitter
 import nextflow.util.BlankSeparatedList
+import nextflow.util.LazyHelper
 /**
  * Models a process file output, which defines a file
  * or set of files to be unstaged from a task work directory.
@@ -36,6 +37,11 @@ class ProcessFileOutput implements PathArityAware {
 
     private Object target
 
+    /**
+     * Flag to support legacy `file` output.
+     */
+    private boolean pathQualifier
+
     /**
      * When true it will not fail if no files are found.
      */
@@ -74,15 +80,20 @@ class ProcessFileOutput implements PathArityAware {
      */
     String type
 
-    ProcessFileOutput(Object target, Map opts) {
+    ProcessFileOutput(Object target, boolean pathQualifier, Map opts) {
         this.target = target
+        this.pathQualifier = pathQualifier
 
         for( Map.Entry entry : opts )
             setProperty(entry.key, entry.value)
     }
 
+    boolean isPathQualifier() {
+        return pathQualifier
+    }
+
     List getFilePatterns(Map context, Path workDir) {
-        final entry = resolve(context, target)
+        final entry = LazyHelper.resolve(context, target)
 
         if( !entry )
             return []
@@ -95,19 +106,13 @@ class ProcessFileOutput implements PathArityAware {
         if( entry instanceof BlankSeparatedList || entry instanceof List )
             return entry.collect( path -> relativize(path.toString(), workDir) )
 
-        // -- literal file name
-        return [ relativize(entry.toString(), workDir) ]
-    }
-
-    protected Object resolve(Map ctx, Object value) {
-
-        if( value instanceof GString )
-            return value.cloneAsLazy(ctx)
+        // -- literal file names separated by ':' (legacy `file` output)
+        final nameString = entry.toString()
+        if( !pathQualifier && nameString.contains(':') )
+            return nameString.split(/:/).collect { String it-> relativize(it, workDir) }
 
-        if( value instanceof Closure )
-            return ctx.with(value)
-
-        return value.toString()
+        // -- literal file name
+        return [ relativize(nameString, workDir) ]
     }
 
     protected String relativize(String path, Path workDir) {
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessInput.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessInput.groovy
index 097eb3f79a..61c3a35116 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessInput.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessInput.groovy
@@ -35,6 +35,11 @@ class ProcessInput implements Cloneable {
 
     private DataflowReadChannel channel
 
+    /**
+     * Flag to support `each` input
+     */
+    private boolean iterator
+
     ProcessInput(String name) {
         this.name = name
     }
@@ -52,27 +57,40 @@ class ProcessInput implements Cloneable {
         if( obj == null )
             throw new IllegalArgumentException('A process input channel evaluates to null')
 
-        final result = obj instanceof Closure
+        final value = obj instanceof Closure
             ? obj.call()
             : obj
 
-        if( result == null )
+        if( value == null )
             throw new IllegalArgumentException('A process input channel evaluates to null')
 
-        def inChannel
-        if ( result instanceof DataflowReadChannel || result instanceof DataflowBroadcast ) {
-            inChannel = CH.getReadChannel(result)
+        if( iterator ) {
+            final result = CH.create()
+            CH.emitAndClose(result, value instanceof Collection ? value : [value])
+            return CH.getReadChannel(result)
         }
-        else {
-            inChannel = CH.value()
-            inChannel.bind(result)
+
+        else if( value instanceof DataflowReadChannel || value instanceof DataflowBroadcast ) {
+            return CH.getReadChannel(value)
         }
 
-        return inChannel
+        else {
+            final result = CH.value()
+            result.bind(value)
+            return result
+        }
     }
 
     DataflowReadChannel getChannel() {
         return channel
     }
 
+    void setIterator(boolean iterator) {
+        this.iterator = iterator
+    }
+
+    boolean isIterator() {
+        return iterator
+    }
+
 }
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessInputs.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessInputs.groovy
index cee0aa7338..d7ac0dd105 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessInputs.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessInputs.groovy
@@ -26,30 +26,60 @@ import groovyx.gpars.dataflow.DataflowReadChannel
 class ProcessInputs implements List, Cloneable {
 
     @Delegate
-    private List target = []
+    private List params = []
 
-    Map env = [:]
+    private Map vars = [:]
 
-    List files = []
+    private Map env = [:]
+
+    private List files = []
 
     Object stdin
 
     @Override
     ProcessInputs clone() {
         def result = (ProcessInputs)super.clone()
-        result.target = new ArrayList<>(target.size())
-        for( ProcessInput param : target ) {
-            result.target.add((ProcessInput)param.clone())
+        result.params = new ArrayList<>(params.size())
+        for( ProcessInput param : params ) {
+            result.params.add((ProcessInput)param.clone())
         }
         return result
     }
 
+    void addParam(String name) {
+        add(new ProcessInput(name))
+    }
+
+    void addVariable(String name, Object value) {
+        vars.put(name, value)
+    }
+
+    void addEnv(String name, Object value) {
+        env.put(name, value)
+    }
+
+    void addFile(ProcessFileInput file) {
+        files.add(file)
+    }
+
     List getNames() {
-        return target*.getName()
+        return params*.getName()
     }
 
     List getChannels() {
-        return target*.getChannel()
+        return params*.getChannel()
+    }
+
+    Map getVariables() {
+        return vars
+    }
+
+    Map getEnv() {
+        return env
+    }
+
+    List getFiles() {
+        return files
     }
 
 }
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessOutput.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessOutput.groovy
index 4f7df7e302..f6d7eab3b3 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessOutput.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessOutput.groovy
@@ -16,17 +16,12 @@
 
 package nextflow.script
 
-import java.nio.file.Path
-
 import groovy.transform.CompileStatic
-import groovy.transform.Memoized
 import groovy.util.logging.Slf4j
 import groovyx.gpars.dataflow.DataflowWriteChannel
-import nextflow.exception.MissingFileException
-import nextflow.exception.MissingValueException
-import nextflow.processor.TaskEnvCollector
-import nextflow.processor.TaskFileCollecter
+import nextflow.processor.TaskOutputCollector
 import nextflow.processor.TaskRun
+import nextflow.util.LazyHelper
 /**
  * Models a process output.
  *
@@ -36,9 +31,7 @@ import nextflow.processor.TaskRun
 @CompileStatic
 class ProcessOutput implements Cloneable {
 
-    static enum Shortcuts { STDOUT }
-
-    private ProcessOutputs parent
+    private ProcessOutputs declaredOutputs
 
     private Object target
 
@@ -48,8 +41,8 @@ class ProcessOutput implements Cloneable {
 
     private DataflowWriteChannel channel
 
-    ProcessOutput(ProcessOutputs parent, Object target, Map opts) {
-        this.parent = parent
+    ProcessOutput(ProcessOutputs declaredOutputs, Object target, Map opts) {
+        this.declaredOutputs = declaredOutputs
         this.target = target
 
         if( opts.name )
@@ -71,102 +64,8 @@ class ProcessOutput implements Cloneable {
     }
 
     Object resolve(TaskRun task) {
-        final ctx = new ResolverContext(parent, optional, task)
-        return resolve0(ctx)
-    }
-
-    private Object resolve0(ResolverContext ctx) {
-        if( target == Shortcuts.STDOUT )
-            return ctx.stdout()
-
-        if( target instanceof Closure )
-            return ctx.with(target)
-
-        return target
+        final ctx = new TaskOutputCollector(declaredOutputs, optional, task)
+        return LazyHelper.resolve(ctx, target)
     }
 
-    static private class ResolverContext {
-
-        private ProcessOutputs parent
-
-        private boolean optional
-
-        private TaskRun task
-
-        ResolverContext(ProcessOutputs parent, boolean optional, TaskRun task) {
-            this.parent = parent
-            this.optional = optional
-            this.task = task
-        }
-
-        /**
-         * Get an environment variable from the task environment.
-         *
-         * @param key
-         */
-        String env(String key) {
-            final varName = parent.env.get(key)
-            final result = env0(task.workDir).get(varName)
-
-            if( result == null && !optional )
-                throw new MissingValueException("Missing environment variable: $varName")
-
-            return result
-        }
-
-        @Memoized
-        static private Map env0(Path workDir) {
-            new TaskEnvCollector(workDir).collect()
-        }
-
-        /**
-         * Get a file or list of files from the task environment.
-         *
-         * @param key
-         */
-        Object path(String key) {
-            final param = parent.files.get(key)
-            final result = new TaskFileCollecter(param, task).collect()
-
-            if( result instanceof Path )
-                task.outputFiles.add(result)
-            else if( result instanceof Collection )
-                task.outputFiles.addAll(result)
-
-            return result
-        }
-
-        /**
-         * Get the standard output from the task environment.
-         */
-        Object stdout() {
-            final result = task.@stdout
-
-            if( result == null && task.type == ScriptType.SCRIPTLET )
-                throw new IllegalArgumentException("Missing 'stdout' for process > ${task.lazyName()}")
-
-            if( result instanceof Path && !result.exists() )
-                throw new MissingFileException("Missing 'stdout' file: ${result.toUriString()} for process > ${task.lazyName()}")
-
-            return result
-        }
-
-        /**
-         * Get a variable from the task context.
-         *
-         * @param name
-         */
-        @Override
-        Object getProperty(String name) {
-            if( name == 'stdout' )
-                return stdout()
-
-            try {
-                return task.context.get(name)
-            }
-            catch( MissingPropertyException e ) {
-                throw new MissingValueException("Missing variable in emit statement: ${e.property}")
-            }
-        }
-    }
 }
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessOutputs.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessOutputs.groovy
index 97e2ab30a9..fa41ebcea6 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessOutputs.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessOutputs.groovy
@@ -18,6 +18,7 @@ package nextflow.script
 
 import groovyx.gpars.dataflow.DataflowQueue
 import groovyx.gpars.dataflow.DataflowWriteChannel
+import nextflow.util.LazyVar
 
 /**
  * Models the process outputs.
@@ -27,33 +28,53 @@ import groovyx.gpars.dataflow.DataflowWriteChannel
 class ProcessOutputs implements List, Cloneable {
 
     @Delegate
-    private List target = []
+    private List params = []
 
-    Map env = [:]
+    private Map env = [:]
 
-    Map files = [:]
+    private Map files = [:]
 
     @Override
     ProcessOutputs clone() {
         def result = (ProcessOutputs)super.clone()
-        result.target = new ArrayList<>(target.size())
-        for( ProcessOutput param : target )
+        result.params = new ArrayList<>(params.size())
+        for( ProcessOutput param : params )
             result.add((ProcessOutput)param.clone())
         return result
     }
 
+    void addParam(Object target, Map opts) {
+        add(new ProcessOutput(this, target, opts))
+    }
+
+    void setDefault() {
+        final param = new ProcessOutput(new LazyVar('stdout'), [:])
+        param.setChannel(new DataflowQueue())
+        params.add(param)
+    }
+
+    void addEnv(String name, Object value) {
+        env.put(name, value)
+    }
+
+    void addFile(String key, ProcessFileOutput file) {
+        files.put(key, file)
+    }
+
     List getNames() {
-        return target*.getName()
+        return params*.getName()
     }
 
     List getChannels() {
-        return target*.getChannel()
+        return params*.getChannel()
     }
 
-    void setDefault() {
-        final param = new ProcessOutput(ProcessOutput.Shortcuts.STDOUT, [:])
-        param.setChannel(new DataflowQueue())
-        target.add(param)
+    Map getEnv() {
+        return env
+    }
+
+    Map getFiles() {
+        return files
     }
 
 }
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ScriptTokens.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ScriptTokens.groovy
index 541f9d455f..63cbecab3d 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ScriptTokens.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ScriptTokens.groovy
@@ -20,20 +20,6 @@ import groovy.transform.CompileStatic
 import groovy.transform.EqualsAndHashCode
 import groovy.transform.ToString
 import groovy.transform.TupleConstructor
-/**
- * Presents a variable definition in the script context.
- *
- * @author Paolo Di Tommaso 
- */
-@ToString
-@EqualsAndHashCode
-@TupleConstructor
-class TokenVar {
-
-    /** The variable name */
-    String name
-
-}
 
 /**
  *  A token used by the DSL to identify a 'file' declaration in a 'tuple' parameter, for example:
@@ -72,26 +58,26 @@ class TokenPathCall {
 }
 
 /**
- * An object of this class replace the {@code stdin} token in input map declaration. For example:
+ * An object of this class replace the {@code stdin} token in input tuple declaration. For example:
  * 
      * input:
- *   map( stdin, .. ) from x
+ *   tuple( stdin, .. ) from x
  * 
    
  *
  * @see nextflow.ast.DslCodeVisitor
- * @see nextflow.script.params.TupleInParam#bind(java.lang.Object[])
+ * @see nextflow.script.dsl.ProcessDsl#_in_tuple(java.lang.Object[])
  */
 class TokenStdinCall { }
 
 /**
- * An object of this class replace the {@code stdout} token in input map declaration. For example:
+ * An object of this class replace the {@code stdout} token in input tuple declaration. For example:
  * 
      * input:
- *   map( stdout, .. ) into x
+ *   tuple( stdout, .. ) into x
  * 
    
  *
  * @see nextflow.ast.DslCodeVisitor
- * @see nextflow.script.params.TupleOutParam#bind(java.lang.Object[])
+ * @see nextflow.script.dsl.ProcessDsl#_out_tuple(java.util.Map,java.lang.Object[])
  */
 class TokenStdoutCall { }
 
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessBuilder.groovy b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessBuilder.groovy
index 63e134a739..4db834b630 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessBuilder.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessBuilder.groovy
@@ -24,7 +24,6 @@ import nextflow.exception.ConfigParseException
 import nextflow.exception.IllegalConfigException
 import nextflow.exception.IllegalDirectiveException
 import nextflow.exception.ScriptRuntimeException
-import nextflow.processor.ConfigList
 import nextflow.processor.ErrorStrategy
 import nextflow.script.ProcessInputs
 import nextflow.script.ProcessOutputs
@@ -32,6 +31,7 @@ import nextflow.script.BaseScript
 import nextflow.script.BodyDef
 import nextflow.script.ProcessConfig
 import nextflow.script.ProcessDef
+import nextflow.util.LazyList
 
 /**
  * Builder for {@link ProcessDef}.
@@ -84,10 +84,13 @@ class ProcessBuilder {
             'time'
     ]
 
-    private BaseScript ownerScript
-    private String processName
-    private BodyDef body
-    private ProcessConfig config
+    protected BaseScript ownerScript
+
+    protected String processName
+
+    protected BodyDef body
+
+    protected ProcessConfig config
 
     ProcessBuilder(BaseScript ownerScript, String processName) {
         this.ownerScript = ownerScript
@@ -260,7 +263,7 @@ class ProcessBuilder {
         // -- get the current label, it must be a list
         def allLabels = (List)config.get('label')
         if( !allLabels ) {
-            allLabels = new ConfigList()
+            allLabels = new LazyList()
             config.put('label', allLabels)
         }
 
@@ -293,7 +296,7 @@ class ProcessBuilder {
 
         def result = (List)config.module
         if( result == null ) {
-            result = new ConfigList()
+            result = new LazyList()
             config.put('module', result)
         }
 
@@ -310,7 +313,7 @@ class ProcessBuilder {
 
         def allOptions = (List)config.get('pod')
         if( !allOptions ) {
-            allOptions = new ConfigList()
+            allOptions = new LazyList()
             config.put('pod', allOptions)
         }
 
@@ -332,7 +335,7 @@ class ProcessBuilder {
 
         def dirs = (List)config.get('publishDir')
         if( !dirs ) {
-            dirs = new ConfigList()
+            dirs = new LazyList()
             config.put('publishDir', dirs)
         }
 
@@ -406,7 +409,7 @@ class ProcessBuilder {
         // -- get the current label, it must be a list
         def allSecrets = (List)config.get('secret')
         if( !allSecrets ) {
-            allSecrets = new ConfigList()
+            allSecrets = new LazyList()
             config.put('secret', allSecrets)
         }
 
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessDsl.groovy b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessDsl.groovy
index ae7129e76c..0bd3381af0 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessDsl.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessDsl.groovy
@@ -16,132 +16,321 @@
 
 package nextflow.script.dsl
 
+import groovy.transform.CompileDynamic
+import groovy.transform.CompileStatic
+import nextflow.processor.TaskOutputCollector
 import nextflow.script.BaseScript
-import nextflow.script.params.*
+import nextflow.script.ProcessDef
+import nextflow.script.ProcessFileInput
+import nextflow.script.ProcessFileOutput
+import nextflow.script.ProcessInputs
+import nextflow.script.ProcessOutput
+import nextflow.script.ProcessOutputs
+import nextflow.script.TokenEnvCall
+import nextflow.script.TokenFileCall
+import nextflow.script.TokenPathCall
+import nextflow.script.TokenStdinCall
+import nextflow.script.TokenStdoutCall
+import nextflow.script.TokenValCall
+import nextflow.util.LazyAware
+import nextflow.util.LazyList
+import nextflow.util.LazyVar
 
 /**
  * Implements the process DSL.
  *
  * @author Paolo Di Tommaso 
+ * @author Ben Sherman 
  */
+@CompileStatic
 class ProcessDsl extends ProcessBuilder {
 
+    private ProcessInputs inputs = new ProcessInputs()
+
+    private ProcessOutputs outputs = new ProcessOutputs()
+
     ProcessDsl(BaseScript ownerScript, String processName) {
         super(ownerScript, processName)
     }
 
     /// INPUTS
 
-    InParam _in_each( Object obj ) {
-        new EachInParam(config).bind(obj)
+    void _in_each(LazyVar var) {
+        _in_val(var)
+        inputs.last().setIterator(true)
+    }
+
+    void _in_each(TokenFileCall file) {
+        _in_file(file.target)
+        inputs.last().setIterator(true)
+    }
+
+    void _in_each(TokenPathCall path) {
+        _in_path(path.target)
+        inputs.last().setIterator(true)
     }
 
-    InParam _in_env( Object obj ) {
-        new EnvInParam(config).bind(obj)
+    void _in_env(LazyVar var) {
+        final param = "\$in${inputs.size()}".toString()
+
+        inputs.addParam(param)
+        inputs.addEnv(var.name, new LazyVar(param))
+    }
+
+    void _in_file(Object source) {
+        final param = _in_path0(source, false, [:])
+        inputs.addParam(param)
     }
 
-    InParam _in_file( Object obj ) {
-        new FileInParam(config).bind(obj)
+    void _in_path(Map opts=[:], Object source) {
+        final param = _in_path0(source, true, opts)
+        inputs.addParam(param)
     }
 
-    InParam _in_path( Map opts=[:], Object obj ) {
-        new FileInParam(config)
-                .setPathQualifier(true)
-                .setOptions(opts)
-                .bind(obj)
+    private String _in_path0(Object source, boolean pathQualifier, Map opts) {
+        if( source instanceof LazyVar ) {
+            final var = (LazyVar)source
+            inputs.addFile(new ProcessFileInput(var, var.name, pathQualifier, opts))
+            return var.name
+        }
+        else if( source instanceof CharSequence ) {
+            final param = "\$in${inputs.size()}"
+            if( !opts.stageAs )
+                opts.stageAs = source.toString()
+            inputs.addFile(new ProcessFileInput(new LazyVar(param), null, pathQualifier, opts))
+            return param
+        }
+        else
+            throw new IllegalArgumentException()
     }
 
-    InParam _in_stdin( Object obj = null ) {
-        def result = new StdInParam(config)
-        if( obj )
-            result.bind(obj)
-        result
+    void _in_stdin(LazyVar var=null) {
+        final param = var != null
+            ? var.name
+            : "\$in${inputs.size()}".toString()
+
+        inputs.addParam(param)
+        inputs.stdin = new LazyVar(param)
     }
 
-    InParam _in_tuple( Object... obj ) {
-        if( obj.length < 2 )
+    @CompileDynamic
+    void _in_tuple(Object... elements) {
+        if( elements.length < 2 )
             throw new IllegalArgumentException("Input `tuple` must define at least two elements -- Check process `$processName`")
-        new TupleInParam(config).bind(obj)
+
+        final param = "\$in${inputs.size()}".toString()
+        inputs.addParam(param)
+
+        for( int i = 0; i < elements.length; i++ ) {
+            final item = elements[i]
+
+            if( item instanceof LazyVar ) {
+                final var = (LazyVar)item
+                throw new IllegalArgumentException("Unqualified input value declaration is not allowed - replace `tuple ${var.name},..` with `tuple val(${var.name}),..`")
+            }
+            else if( item instanceof TokenValCall && item.val instanceof LazyVar ) {
+                inputs.addVariable(item.val.name, new LazyTupleElement(param, i))
+            }
+            else if( item instanceof TokenEnvCall && item.val instanceof LazyVar ) {
+                inputs.addEnv(item.val.name, new LazyTupleElement(param, i))
+            }
+            else if( item instanceof TokenFileCall ) {
+                final name = _in_path0(item.target, false, [:])
+                inputs.addVariable(name, new LazyTupleElement(param, i))
+            }
+            else if( item instanceof TokenPathCall ) {
+                final name = _in_path0(item.target, true, item.opts)
+                inputs.addVariable(name, new LazyTupleElement(param, i))
+            }
+            else if( item instanceof Map ) {
+                throw new IllegalArgumentException("Unqualified input file declaration is not allowed - replace `tuple $item,..` with `tuple path(${item.key}, stageAs:'${item.value}'),..`")
+            }
+            else if( item instanceof GString ) {
+                throw new IllegalArgumentException("Unqualified input file declaration is not allowed - replace `tuple \"$item\".. with `tuple path(\"$item\")..`")
+            }
+            else if( item instanceof TokenStdinCall || item == '-' ) {
+                inputs.stdin = new LazyTupleElement(param, i)
+            }
+            else if( item instanceof String ) {
+                throw new IllegalArgumentException("Unqualified input file declaration is not allowed - replace `tuple '$item',..` with `tuple path('$item'),..`")
+            }
+            else
+                throw new IllegalArgumentException()
+        }
     }
 
-    InParam _in_val( Object obj ) {
-        new ValueInParam(config).bind(obj)
+    void _in_val(LazyVar var) {
+        inputs.addParam(var.name)
     }
 
     /// OUTPUTS
 
-    OutParam _out_env( Object obj ) {
-        new EnvOutParam(config)
-                .bind(obj)
-    }
+    void _out_env(Map opts=[:], LazyVar var) {
+        if( opts.emit )
+            opts.name = opts.remove('emit')
 
-    OutParam _out_env( Map opts, Object obj ) {
-        new EnvOutParam(config)
-                .setOptions(opts)
-                .bind(obj)
+        outputs.addEnv(var.name, var.name)
+        outputs.addParam(new LazyEnvCall(var.name), opts)
     }
 
-    OutParam _out_file( Object obj ) {
+    void _out_file(Object target) {
         // note: check that is a String type to avoid to force
         // the evaluation of GString object to a string
-        if( obj instanceof String && obj == '-' )
-            new StdOutParam(config).bind(obj)
-        else
-            new FileOutParam(config).bind(obj)
+        if( target instanceof String && target == '-' ) {
+            _out_stdout()
+            return
+        }
+
+        final key = _out_path0(target, false, [:])
+        outputs.addParam(new LazyPathCall(key), [:])
     }
 
-    OutParam _out_path( Map opts=null, Object obj ) {
+    void _out_path(Map opts=[:], Object target) {
         // note: check that is a String type to avoid to force
         // the evaluation of GString object to a string
-        if( obj instanceof String && obj == '-' )
-            new StdOutParam(config)
-                    .setOptions(opts)
-                    .bind(obj)
+        if( target instanceof String && target == '-' ) {
+            _out_stdout(opts)
+            return
+        }
 
-        else
-            new FileOutParam(config)
-                    .setPathQualifier(true)
-                    .setOptions(opts)
-                    .bind(obj)
+        // separate param options from path options
+        final paramOpts = [optional: opts.optional]
+        if( opts.emit )
+            paramOpts.name = opts.remove('emit')
+
+        final key = _out_path0(target, true, opts)
+        outputs.addParam(new LazyPathCall(key), paramOpts)
     }
 
-    OutParam _out_stdout( Map opts ) {
-        new StdOutParam(config)
-                .setOptions(opts)
-                .bind('-')
+    private String _out_path0(Object target, boolean pathQualifier, Map opts) {
+        final key = "\$file${outputs.getFiles().size()}".toString()
+        outputs.addFile(key, new ProcessFileOutput(target, pathQualifier, opts))
+        return key
     }
 
-    OutParam _out_stdout( obj = null ) {
-        def result = new StdOutParam(config).bind('-')
-        if( obj )
-            result.setInto(obj)
-        result
+    void _out_stdout(Map opts=[:]) {
+        if( opts.emit )
+            opts.name = opts.remove('emit')
+
+        outputs.addParam(new LazyVar('stdout'), opts)
     }
 
-    OutParam _out_tuple( Object... obj ) {
-        if( obj.length < 2 )
+    @CompileDynamic
+    void _out_tuple(Map opts=[:], Object... elements) {
+        if( elements.length < 2 )
             throw new IllegalArgumentException("Output `tuple` must define at least two elements -- Check process `$processName`")
-        new TupleOutParam(config)
-                .bind(obj)
+
+        // separate param options from path options
+        final paramOpts = [optional: opts.optional]
+        if( opts.emit )
+            paramOpts.name = opts.remove('emit')
+
+        // make lazy list with tuple elements
+        final target = new LazyList(elements.size())
+
+        for( int i = 0; i < elements.length; i++ ) {
+            final item = elements[i]
+
+            if( item instanceof LazyVar ) {
+                throw new IllegalArgumentException("Unqualified output value declaration is not allowed - replace `tuple ${item.name},..` with `tuple val(${item.name}),..`")
+            }
+            else if( item instanceof TokenValCall ) {
+                target << item.val
+            }
+            else if( item instanceof TokenEnvCall && item.val instanceof LazyVar ) {
+                final var = (LazyVar)item.val
+                outputs.addEnv(var.name, var.name)
+                target << new LazyEnvCall(var.name)
+            }
+            else if( item instanceof TokenFileCall ) {
+                // file pattern can be a String or GString
+                final key = _out_path0(item.target, false, [:])
+                target << new LazyPathCall(key)
+            }
+            else if( item instanceof TokenPathCall ) {
+                // file pattern can be a String or GString
+                final key = _out_path0(item.target, true, item.opts)
+                target << new LazyPathCall(key)
+            }
+            else if( item instanceof GString ) {
+                throw new IllegalArgumentException("Unqualified output path declaration is not allowed - replace `tuple \"$item\",..` with `tuple path(\"$item\"),..`")
+            }
+            else if( item instanceof TokenStdoutCall || item == '-' ) {
+                target << new LazyVar('stdout')
+            }
+            else if( item instanceof String ) {
+                throw new IllegalArgumentException("Unqualified output path declaration is not allowed - replace `tuple '$item',..` with `tuple path('$item'),..`")
+            }
+            else
+                throw new IllegalArgumentException("Invalid `tuple` output parameter declaration -- item: ${item}")
+        }
+
+        outputs.addParam(target, paramOpts)
     }
 
-    OutParam _out_tuple( Map opts, Object... obj ) {
-        if( obj.length < 2 )
-            throw new IllegalArgumentException("Output `tuple` must define at least two elements -- Check process `$processName`")
-        new TupleOutParam(config)
-                .setOptions(opts)
-                .bind(obj)
+    void _out_val(Map opts=[:], Object target) {
+        outputs.addParam(target, opts)
+    }
+
+    /// BUILD
+
+    ProcessDef build() {
+        config.setInputs(inputs)
+        config.setOutputs(outputs)
+        super.build()
     }
 
-    OutParam _out_val( Object obj ) {
-        new ValueOutParam(config)
-                .bind(obj)
+}
+
+@CompileStatic
+class LazyTupleElement extends LazyVar {
+    int index
+
+    LazyTupleElement(String name, int index) {
+        super(name)
+        this.index = index
     }
 
-    OutParam _out_val( Map opts, Object obj ) {
-        new ValueOutParam(config)
-                .setOptions(opts)
-                .bind(obj)
+    @Override
+    Object resolve(Object binding) {
+        final tuple = super.resolve(binding)
+        if( tuple instanceof List )
+            return tuple[index]
+        else
+            throw new IllegalArgumentException("Lazy binding of `${name}[${index}]` failed because `${name}` is not a tuple")
     }
+}
+
+@CompileStatic
+class LazyEnvCall implements LazyAware {
+    String key
 
+    LazyEnvCall(String key) {
+        this.key = key
+    }
+
+    @Override
+    Object resolve(Object binding) {
+        if( binding !instanceof TaskOutputCollector )
+            throw new IllegalStateException()
+
+        ((TaskOutputCollector)binding).env(key)
+    }
+}
+
+@CompileStatic
+class LazyPathCall implements LazyAware {
+    String key
+
+    LazyPathCall(String key) {
+        this.key = key
+    }
+
+    @Override
+    Object resolve(Object binding) {
+        if( binding !instanceof TaskOutputCollector )
+            throw new IllegalStateException()
+
+        ((TaskOutputCollector)binding).path(key)
+    }
 }
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessInputsBuilder.groovy b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessInputsBuilder.groovy
index 227c932d47..ea37a702e1 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessInputsBuilder.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessInputsBuilder.groovy
@@ -32,12 +32,12 @@ class ProcessInputsBuilder {
     private ProcessInputs inputs = new ProcessInputs()
 
     ProcessInputsBuilder env(String name, Object source) {
-        inputs.env.put(name, source)
+        inputs.addEnv(name, source)
         return this
     }
 
     ProcessInputsBuilder path(Map opts=[:], Object source) {
-        inputs.files.add(new ProcessFileInput(source, null, true, opts))
+        inputs.addFile(new ProcessFileInput(source, null, true, opts))
         return this
     }
 
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessOutputsBuilder.groovy b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessOutputsBuilder.groovy
index 52b1e72f64..ada10c6bbf 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessOutputsBuilder.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessOutputsBuilder.groovy
@@ -35,8 +35,8 @@ class ProcessOutputsBuilder {
         env(name, name)
     }
 
-    ProcessOutputsBuilder env(String name, String target) {
-        outputs.env.put(name, target)
+    ProcessOutputsBuilder env(String key, String target) {
+        outputs.addEnv(key, target)
         return this
     }
 
@@ -44,8 +44,8 @@ class ProcessOutputsBuilder {
         path(opts, name, name)
     }
 
-    ProcessOutputsBuilder path(Map opts=[:], String name, Object target) {
-        outputs.files.put(name, new ProcessFileOutput(target, opts))
+    ProcessOutputsBuilder path(Map opts=[:], String key, Object target) {
+        outputs.addFile(key, new ProcessFileOutput(target, true, opts))
         return this
     }
 
@@ -54,18 +54,18 @@ class ProcessOutputsBuilder {
      * be evaluated after the task execution. For example:
      *
      *   env 'SAMPLE_ID'           // declare output env 'SAMPLE_ID'
-     *   path '$out0', 'file.txt'  // declare output file 'file.txt'
+     *   path '$file0', 'file.txt' // declare output file 'file.txt'
      *
      *   emit { sample_id }        // variable 'sample_id' in task context
      *   emit { stdout }           // standard output of task script
-     *   emit { [env('SAMPLE_ID'), path('$out0')] }
-     *   emit { new Sample(sample_id, path('$out0')) }
+     *   emit { [env('SAMPLE_ID'), path('$file0')] }
+     *   emit { new Sample(sample_id, path('$file0')) }
      *
      * @param opts
      * @param target
      */
     ProcessOutputsBuilder emit(Map opts=[:], Object target) {
-        outputs.add(new ProcessOutput(outputs, target, opts))
+        outputs.addParam(target, opts)
         return this
     }
 
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/BaseInParam.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/BaseInParam.groovy
deleted file mode 100644
index 43c2ae70da..0000000000
--- a/modules/nextflow/src/main/groovy/nextflow/script/params/BaseInParam.groovy
+++ /dev/null
@@ -1,200 +0,0 @@
-/*
- * Copyright 2013-2023, Seqera Labs
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package nextflow.script.params
-
-import groovy.util.logging.Slf4j
-import groovyx.gpars.dataflow.DataflowBroadcast
-import groovyx.gpars.dataflow.DataflowQueue
-import groovyx.gpars.dataflow.DataflowReadChannel
-import nextflow.NF
-import nextflow.exception.ProcessException
-import nextflow.exception.ScriptRuntimeException
-import nextflow.extension.CH
-import nextflow.script.ProcessConfig
-import nextflow.script.TokenVar
-/**
- * Model a process generic input parameter
- *
- * @author Paolo Di Tommaso 
- */
-
-@Slf4j
-abstract class BaseInParam extends BaseParam implements InParam {
-
-    protected fromObject
-
-    protected bindObject
-
-    protected owner
-
-    /**
-     * The channel to which the input value is bound
-     */
-    private inChannel
-
-    /**
-     * @return The input channel instance used by this parameter to receive the process inputs
-     */
-    DataflowReadChannel getInChannel() {
-        init()
-        return inChannel
-    }
-
-    def getBindObject() {
-        bindObject
-    }
-
-    BaseInParam( ProcessConfig config ) {
-        this(config.getOwnerScript().getBinding(), config.getInputs())
-    }
-
-    /**
-     * @param script The global script object
-     * @param obj
-     */
-    BaseInParam( Binding binding, List holder, short ownerIndex = -1 ) {
-        super(binding,holder,ownerIndex)
-    }
-
-    abstract String getTypeName()
-
-    protected DataflowReadChannel inputValToChannel( value ) {
-        checkFromNotNull(value)
-
-        if ( value instanceof DataflowReadChannel || value instanceof DataflowBroadcast )  {
-            return CH.getReadChannel(value)
-        }
-
-        final result = CH.value()
-        result.bind(value)
-        return result
-    }
-
-
-    /**
-     * Lazy parameter initializer.
-     *
-     * @return The parameter object itself
-     */
-    @Override
-    protected void lazyInit() {
-
-        if( fromObject == null && (bindObject == null || bindObject instanceof GString || bindObject instanceof Closure ) ) {
-            throw new IllegalStateException("Missing 'bind' declaration in input parameter")
-        }
-
-        // fallback on the bind object if the 'fromObject' is not defined
-        if( fromObject == null ) {
-            fromObject = bindObject
-        }
-
-        // initialize the *inChannel* object based on the 'target' attribute
-        def result
-        if( fromObject instanceof TokenVar ) {
-            // when the value is a variable reference
-            // - use that name for the parameter itself
-            // - get the variable value in the script binding
-            result = getScriptVar(fromObject.name)
-        }
-        else if( fromObject instanceof Closure ) {
-            result = fromObject.call()
-        }
-        else {
-            result = fromObject
-        }
-
-        inChannel = inputValToChannel(result)
-    }
-
-    /**
-     * @return The parameter name
-     */
-    String getName() {
-        if( bindObject instanceof TokenVar )
-            return bindObject.name
-
-        if( bindObject instanceof String )
-            return bindObject
-
-        if( bindObject instanceof Closure )
-            return '__$' + this.toString()
-
-        throw new IllegalArgumentException("Invalid process input definition")
-    }
-
-    BaseInParam bind( Object obj ) {
-        this.bindObject = obj
-        return this
-    }
-
-    private void checkFromNotNull(obj) {
-        if( obj != null ) return
-        def message = 'A process input channel evaluates to null'
-        def name = null
-        if( bindObject instanceof TokenVar )
-            name = bindObject.name
-        else if( bindObject instanceof CharSequence )
-            name = bindObject.toString()
-        if( name )
-            message += " -- Invalid declaration `${getTypeName()} $name`"
-        throw new IllegalArgumentException(message)
-    }
-
-    void setFrom( obj ) {
-        checkFromNotNull(obj)
-        fromObject = obj
-    }
-
-    Object getRawChannel() {
-        if( CH.isChannel(fromObject) )
-            return fromObject
-        if( CH.isChannel(inChannel) )
-            return inChannel
-        throw new IllegalStateException("Missing input channel")
-    }
-
-    def decodeInputs( List inputs ) {
-        final UNDEF = -1 as short
-        def value = inputs[index]
-
-        if( mapIndex == UNDEF || owner instanceof EachInParam )
-            return value
-
-        if( mapIndex != UNDEF ) {
-            def result
-            if( value instanceof Map ) {
-                result = value.values()
-            }
-            else if( value instanceof Collection ) {
-                result = value
-            }
-            else {
-                result = [value]
-            }
-
-            try {
-                return result[mapIndex]
-            }
-            catch( IndexOutOfBoundsException e ) {
-                throw new ProcessException(e)
-            }
-        }
-
-        return value
-    }
-
-}
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/BaseOutParam.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/BaseOutParam.groovy
deleted file mode 100644
index 366110ded4..0000000000
--- a/modules/nextflow/src/main/groovy/nextflow/script/params/BaseOutParam.groovy
+++ /dev/null
@@ -1,185 +0,0 @@
-/*
- * Copyright 2013-2023, Seqera Labs
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package nextflow.script.params
-
-import groovy.transform.PackageScope
-import groovy.util.logging.Slf4j
-import groovyx.gpars.dataflow.DataflowWriteChannel
-import nextflow.NF
-import nextflow.extension.CH
-import nextflow.script.ProcessConfig
-import nextflow.script.TokenVar
-import nextflow.util.ConfigHelper
-/**
- * Model a process generic output parameter
- *
- * @author Paolo Di Tommaso 
- */
-@Slf4j
-abstract class BaseOutParam extends BaseParam implements OutParam {
-
-    /** The out parameter name */
-    protected String nameObj
-
-    protected intoObj
-
-    protected List outChannels = new ArrayList<>(10)
-
-    @PackageScope
-    boolean singleton
-
-    String channelEmitName
-
-    BaseOutParam( Binding binding, List list, short ownerIndex = -1) {
-        super(binding,list,ownerIndex)
-    }
-
-    BaseOutParam( ProcessConfig config ) {
-        super(config.getOwnerScript().getBinding(), config.getOutputs())
-    }
-
-    Object clone() {
-        final copy = (BaseOutParam)super.clone()
-        copy.outChannels = new ArrayList<>(10)
-        return copy
-    }
-
-    void lazyInit() {
-
-        if( intoObj instanceof TokenVar || intoObj instanceof TokenVar[] ) {
-            throw new IllegalArgumentException("Not a valid output channel argument: $intoObj")
-        }
-        else if( intoObj != null ) {
-            lazyInitImpl(intoObj)
-        }
-        else if( nameObj instanceof String ) {
-            lazyInitImpl(nameObj)
-        }
-
-    }
-
-    @PackageScope
-    void setSingleton( boolean value ) {
-        this.singleton = value
-    }
-
-    @PackageScope
-    void lazyInitImpl( def target ) {
-        final channel = (target != null)
-            ? outputValToChannel(target)
-            : null
-
-        if( channel ) {
-            outChannels.add(channel)
-        }
-    }
-
-    /**
-     * Creates a channel variable in the script context
-     *
-     * @param channel it can be a string representing a channel variable name in the script context. If
-     *      the variable does not exist it creates a {@code DataflowVariable} in the script with that name.
-     *      If the specified {@code value} is a {@code DataflowWriteChannel} object, use this object
-     *      as the output channel
-     *
-     * @param factory The type of the channel to create, either {@code DataflowVariable} or {@code DataflowQueue}
-     * @return The created (or specified) channel instance
-     */
-    final protected DataflowWriteChannel outputValToChannel( Object channel ) {
-
-        if( channel instanceof String ) {
-            // the channel is specified by name
-            def local = channel
-
-            // look for that name in the 'script' context
-            channel = binding.hasVariable(local) ? binding.getVariable(local) : null
-            if( channel instanceof DataflowWriteChannel ) {
-                // that's OK -- nothing to do
-            }
-            else {
-                if( channel == null ) {
-                    log.trace "Creating new output channel > $local"
-                }
-                else {
-                    log.warn "Output channel `$local` overrides another variable with the same name declared in the script context -- Rename it to avoid possible conflicts"
-                }
-
-                // instantiate the new channel
-                channel = CH.create( singleton )
-
-                // bind it to the script on-fly
-                if( local != '-' && binding ) {
-                    // bind the outputs to the script scope
-                    binding.setVariable(local, channel)
-                }
-            }
-        }
-
-        if( channel instanceof DataflowWriteChannel ) {
-            return channel
-        }
-
-        throw new IllegalArgumentException("Invalid output channel reference")
-    }
-
-
-    BaseOutParam bind( def obj ) {
-        if( obj instanceof TokenVar )
-            this.nameObj = obj.name
-
-        else
-            this.nameObj = ( obj?.toString() ?: null )
-
-        return this
-    }
-
-    void setInto( Object obj ) {
-        intoObj = obj
-    }
-
-    DataflowWriteChannel getOutChannel() {
-        init()
-        return outChannels ? outChannels.get(0) : null
-    }
-
-    String getName() {
-        if( nameObj != null )
-            return nameObj.toString()
-        throw new IllegalStateException("Missing 'name' property in output parameter")
-    }
-
-    @Override
-    BaseOutParam setOptions(Map opts) {
-        super.setOptions(opts)
-        return this
-    }
-
-    BaseOutParam setEmit( value ) {
-        if( isNestedParam() )
-            throw new IllegalArgumentException("Output `emit` option is not allowed in tuple components")
-        if( !value )
-            throw new IllegalArgumentException("Missing output `emit` name")
-        if( !ConfigHelper.isValidIdentifier(value) ) {
-            final msg = "Output emit '$value' is not a valid name -- Make sure it starts with an alphabetic or underscore character and it does not contain any blank, dot or other special characters"
-            if( NF.strictMode )
-                throw new IllegalArgumentException(msg)
-            log.warn(msg)
-        }
-        this.channelEmitName = value
-        return this
-    }
-}
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/BaseParam.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/BaseParam.groovy
deleted file mode 100644
index 4a5165f8df..0000000000
--- a/modules/nextflow/src/main/groovy/nextflow/script/params/BaseParam.groovy
+++ /dev/null
@@ -1,170 +0,0 @@
-/*
- * Copyright 2013-2023, Seqera Labs
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package nextflow.script.params
-
-import groovy.util.logging.Slf4j
-import nextflow.exception.ScriptRuntimeException
-import nextflow.script.TokenVar
-/**
- * Base class for input/output parameters
- *
- * @author Paolo Di Tommaso 
- */
-@Slf4j
-abstract class BaseParam implements Cloneable {
-
-    /**
-     * The binding context to resolve param variables
-     */
-    final protected Binding binding
-
-    protected List holder
-
-    /**
-     * The param declaration index in the input/output block
-     * Note the index do not change for nested parameters ie. declared in the same tuple param
-     */
-    final short index
-
-    /**
-     * The nested index of tuple composed parameters or -1 when it's a top level param ie. not a tuple element
-     */
-    final short mapIndex
-
-    private boolean initialized
-
-    BaseParam ( Binding binding, List holder, int ownerIndex = -1 ) {
-        this.binding = binding
-        this.holder = holder
-
-        /*
-         * by default the index is got from 'holder' current size
-         * and the mapIndex is =1 (not defined)
-         */
-        if( ownerIndex == -1 ) {
-            index = holder.size()
-            mapIndex = -1
-        }
-
-        /*
-         * when the owner index is provided (not -1) it is used as
-         * the main index and the map index is got from the 'holder' size
-         */
-        else {
-            index = ownerIndex
-            mapIndex = holder.size()
-        }
-
-        // add the the param to the holder list
-        holder.add(this)
-    }
-
-    @Override
-    Object clone() {
-        final copy = (BaseParam)super.clone()
-        copy.holder = this.holder!=null ? new ArrayList(holder) : new ArrayList()
-        return copy
-    }
-
-    String toString() {
-        def p = mapIndex == -1 ? index : "$index:$mapIndex"
-        return "${getTypeSimpleName()}<$p>"
-    }
-
-    String getTypeSimpleName() {
-        this.class.simpleName.toLowerCase()
-    }
-
-    /**
-     * Lazy initializer
-     */
-    protected abstract void lazyInit()
-
-    /**
-     * Initialize the parameter fields if needed
-     */
-    final void init() {
-        if( initialized ) return
-        lazyInit()
-
-        // flag as initialized
-        initialized = true
-    }
-
-
-    /**
-     * Get the value of variable {@code name} in the script context
-     *
-     * @param name The variable name
-     * @param strict If {@code true} raises a {@code MissingPropertyException} when the specified variable does not exist
-     * @return The variable object
-     */
-    protected getScriptVar(String name, boolean strict ) {
-        if( binding.hasVariable(name) ) {
-            return binding.getVariable(name)
-        }
-
-        if( strict )
-            throw new MissingPropertyException(name,this.class)
-
-        return null
-    }
-
-    protected getScriptVar( String name ) {
-        getScriptVar(name,true)
-    }
-
-    protected BaseParam setOptions(Map opts) {
-        if( !opts )
-            return this
-
-        for( Map.Entry entry : opts ) {
-            setProperty(entry.key, entry.value)
-        }
-        return this
-    }
-
-    boolean isNestedParam() {
-        return mapIndex >= 0
-    }
-
-    /**
-     * Report missing method calls as possible syntax errors.
-     */
-    def methodMissing( String name, def args ) {
-        throw new ScriptRuntimeException("Invalid function call `${name}(${argsToString0(args)})` -- possible syntax error")
-    }
-
-    private String argsToString0(args) {
-        if( args instanceof Object[] )
-            args = Arrays.asList(args)
-        if( args instanceof List ) {
-            final result = new ArrayList()
-            for( def it : args )
-                result.add(argsToString1(it))
-            return result.join(',')
-        }
-        return argsToString1(args)
-    }
-
-    private String argsToString1(arg) {
-        if( arg instanceof TokenVar )
-            return arg.name
-        else
-            return String.valueOf((Object)arg)
-    }
-}
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/EachInParam.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/EachInParam.groovy
deleted file mode 100644
index 1ef40a1f6c..0000000000
--- a/modules/nextflow/src/main/groovy/nextflow/script/params/EachInParam.groovy
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Copyright 2013-2023, Seqera Labs
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package nextflow.script.params
-
-import groovy.transform.InheritConstructors
-import groovy.transform.PackageScope
-import groovy.util.logging.Slf4j
-import groovyx.gpars.dataflow.DataflowReadChannel
-import nextflow.extension.CH
-import nextflow.script.TokenFileCall
-import nextflow.script.TokenPathCall
-
-/**
- * Represents a process input *iterator* parameter
- *
- * @author Paolo Di Tommaso 
- */
-@InheritConstructors
-@Slf4j
-class EachInParam extends BaseInParam {
-
-    @Override String getTypeName() { 'each' }
-
-    private List inner = []
-
-    String getName() { '__$'+this.toString() }
-
-    Object clone() {
-        final copy = (EachInParam)super.clone()
-        copy.@inner = new ArrayList<>(inner.size())
-        for( BaseInParam p : inner ) {
-            copy.@inner.add((BaseInParam)p.clone())
-        }
-        return copy
-    }
-
-    EachInParam bind( def obj ) {
-        final nested = createNestedParam(obj)
-        nested.owner = this
-        this.bindObject = nested.bindObject
-        return this
-    }
-
-    protected BaseInParam createNestedParam(obj) {
-        if( obj instanceof TokenFileCall ) {
-            return new FileInParam(binding, inner, index)
-                    .bind(obj.target)
-        }
-        
-        if( obj instanceof TokenPathCall ) {
-            return new FileInParam(binding, inner, index)
-                    .setPathQualifier(true)
-                    .bind(obj.target)
-        }
-
-        return new ValueInParam(binding, inner, index)
-                .bind(obj)
-    }
-
-    InParam getInner() { inner[0] }
-
-    @Override
-    protected DataflowReadChannel inputValToChannel( value ) {
-        def variable = normalizeToVariable( value )
-        super.inputValToChannel(variable)
-    }
-
-    @PackageScope
-    DataflowReadChannel normalizeToVariable( value ) {
-        if( value instanceof Collection ) {
-            final result = CH.create()
-            CH.emitAndClose(result, value as List)
-            return CH.getReadChannel(result)
-        }
-        else
-            return CH.getReadChannel(value)
-    }
-
-}
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/EnvInParam.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/EnvInParam.groovy
deleted file mode 100644
index f5d784a878..0000000000
--- a/modules/nextflow/src/main/groovy/nextflow/script/params/EnvInParam.groovy
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Copyright 2013-2023, Seqera Labs
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package nextflow.script.params
-
-import groovy.transform.InheritConstructors
-
-
-/**
- * Represents a process *environment* input parameter
- *
- * @author Paolo Di Tommaso 
- */
-@InheritConstructors
-class EnvInParam extends BaseInParam {
-
-    @Override
-    String getTypeName() { 'env' }
-
-}
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/EnvOutParam.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/EnvOutParam.groovy
deleted file mode 100644
index 812c739b90..0000000000
--- a/modules/nextflow/src/main/groovy/nextflow/script/params/EnvOutParam.groovy
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Copyright 2013-2023, Seqera Labs
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package nextflow.script.params
-
-import groovy.transform.InheritConstructors
-import nextflow.script.TokenVar
-
-/**
- * Model process `output: env PARAM` definition
- *
- * @author Paolo Di Tommaso 
- */
-@InheritConstructors
-class EnvOutParam extends BaseOutParam implements OptionalParam {
-
-    protected target
-
-    String getName() {
-        return nameObj ? super.getName() : null
-    }
-
-    BaseOutParam bind( def obj ) {
-        // the target value object
-        target = obj
-
-        // retrieve the variable name to be used to fetch the value
-        if( obj instanceof TokenVar ) {
-            this.nameObj = obj.name
-        }
-
-        return this
-    }
-
-}
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/FileInParam.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/FileInParam.groovy
deleted file mode 100644
index ce9c984d7c..0000000000
--- a/modules/nextflow/src/main/groovy/nextflow/script/params/FileInParam.groovy
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Copyright 2013-2023, Seqera Labs
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package nextflow.script.params
-
-import groovy.transform.InheritConstructors
-import groovy.util.logging.Slf4j
-
-/**
- * Represents a process *file* input parameter
- *
- * @author Paolo Di Tommaso 
- */
-@Slf4j
-@InheritConstructors
-class FileInParam extends BaseInParam implements PathQualifier {
-
-    private boolean pathQualifier
-
-    private Map options
-
-    @Override String getTypeName() { pathQualifier ? 'path' : 'file' }
-
-    @Override String getTypeSimpleName() { getTypeName() + "inparam" }
-
-    String getName() {
-        if( bindObject instanceof Map ) {
-            assert !pathQualifier
-            def entry = bindObject.entrySet().first()
-            return entry?.key
-        }
-
-        if( bindObject instanceof GString ) {
-            return '__$' + this.toString()
-        }
-
-        return super.getName()
-    }
-
-    @Override
-    BaseInParam bind( obj ) {
-        if( pathQualifier && obj instanceof Map )
-            throw new IllegalArgumentException("Input `path` does not allow such arguments: ${obj.entrySet().collect{"${it.key}:${it.value}"}.join(',')}")
-        super.bind(obj)
-        return this
-    }
-
-    @Override
-    FileInParam setPathQualifier(boolean flag) {
-        pathQualifier = flag
-        return this
-    }
-
-    @Override
-    boolean isPathQualifier() { pathQualifier }
-
-    @Override
-    FileInParam setOptions(Map opts) {
-        this.options = opts
-        return this
-    }
-
-    Map getOptions() { options }
-
-}
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/FileOutParam.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/FileOutParam.groovy
deleted file mode 100644
index 8b62db0c3f..0000000000
--- a/modules/nextflow/src/main/groovy/nextflow/script/params/FileOutParam.groovy
+++ /dev/null
@@ -1,215 +0,0 @@
-/*
- * Copyright 2013-2023, Seqera Labs
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package nextflow.script.params
-
-import java.nio.file.Path
-
-import groovy.transform.InheritConstructors
-import groovy.transform.PackageScope
-import groovy.util.logging.Slf4j
-import nextflow.NF
-import nextflow.exception.IllegalFileException
-import nextflow.file.FilePatternSplitter
-import nextflow.script.PathArityAware
-import nextflow.script.TokenVar
-import nextflow.util.BlankSeparatedList
-/**
- * Model a process *file* output parameter
- *
- * @author Paolo Di Tommaso 
- */
-@Slf4j
-@InheritConstructors
-class FileOutParam extends BaseOutParam implements OutParam, OptionalParam, PathArityAware, PathQualifier {
-
-    /**
-     * ONLY FOR TESTING DO NOT USE
-     */
-    protected FileOutParam(Map params) {
-        super(new Binding(), [])
-    }
-
-    /**
-     * The character used to separate multiple names (pattern) in the output specification
-     *
-     * This is only used by `file` qualifier. It's not supposed to be used anymore
-     * by the new `path` qualifier.
-     *
-     */
-    @Deprecated
-    String separatorChar = ':'
-
-    /**
-     * When {@code true} star wildcard (*) matches hidden files (files starting with a dot char)
-     * By default it does not, coherently with linux bash rule
-     */
-    boolean hidden
-
-    /**
-     * When {@code true} file pattern includes input files as well as output files.
-     * By default a file pattern matches only against files produced by the process, not
-     * the ones received as input
-     */
-    boolean includeInputs
-
-    /**
-     * The type of path to output, either {@code file}, {@code dir} or {@code any}
-     */
-    String type
-
-    /**
-     * Maximum number of directory levels to visit (default: no limit)
-     */
-    Integer maxDepth
-
-    /**
-     * When true it follows symbolic links during directories tree traversal, otherwise they are managed as files (default: true)
-     */
-    boolean followLinks = true
-
-    boolean glob = true
-
-    private GString gstring
-    private Closure dynamicObj
-    private String filePattern
-    private boolean pathQualifier
-
-    /**
-     * @return {@code true} when the file name is parametric i.e contains a variable name to be resolved, {@code false} otherwise
-     */
-    boolean isDynamic() { dynamicObj || gstring != null }
-
-    @Override
-    BaseOutParam bind( obj ) {
-
-        if( obj instanceof GString ) {
-            gstring = obj
-            return this
-        }
-
-        if( obj instanceof TokenVar ) {
-            this.nameObj = obj.name
-            dynamicObj = { delegate.containsKey(obj.name) ? delegate.get(obj.name): obj.name }
-            return this
-        }
-
-        if( obj instanceof Closure ) {
-            dynamicObj = obj
-            return this
-        }
-
-        this.filePattern = obj.toString()
-        return this
-    }
-
-    List getFilePatterns(Map context, Path workDir) {
-
-        def entry = null
-        if( dynamicObj ) {
-            entry = context.with(dynamicObj)
-        }
-        else if( gstring != null ) {
-            def strict = (getName() == null)
-            try {
-                entry = gstring.cloneAsLazy(context)
-            }
-            catch( MissingPropertyException e ) {
-                if( strict )
-                    throw e
-            }
-        }
-        else {
-            entry = filePattern
-        }
-
-        if( !entry )
-            return []
-
-        if( entry instanceof Path )
-            return [ relativize(entry, workDir) ]
-
-        // handle a collection of files
-        if( entry instanceof BlankSeparatedList || entry instanceof List ) {
-            return entry.collect { relativize(it.toString(), workDir) }
-        }
-
-        // normalize to a string object
-        final nameString = entry.toString()
-        if( separatorChar && nameString.contains(separatorChar) ) {
-            return nameString.split(/\${separatorChar}/).collect { String it-> relativize(it, workDir) }
-        }
-
-        return [relativize(nameString, workDir)]
-
-    }
-
-    @PackageScope String getFilePattern() { filePattern }
-
-    @PackageScope
-    String relativize(String path, Path workDir) {
-        if( !path.startsWith('/') )
-            return path
-
-        final dir = workDir.toString()
-        if( !path.startsWith(dir) )
-            throw new IllegalFileException("File `$path` is outside the scope of the process work directory: $workDir")
-
-        if( path.length()-dir.length()<2 )
-            throw new IllegalFileException("Missing output file name")
-
-        return path.substring(dir.size()+1)
-    }
-
-    @PackageScope
-    String relativize(Path path, Path workDir) {
-        if( !path.isAbsolute() )
-            return glob ? FilePatternSplitter.GLOB.escape(path) : path
-
-        if( !path.startsWith(workDir) )
-            throw new IllegalFileException("File `$path` is outside the scope of the process work directory: $workDir")
-
-        if( path.nameCount == workDir.nameCount )
-            throw new IllegalFileException("Missing output file name")
-
-        final rel = path.subpath(workDir.getNameCount(), path.getNameCount())
-        return glob ? FilePatternSplitter.GLOB.escape(rel) : rel
-    }
-
-    /**
-     * Override the default to allow null as a value name
-     * @return
-     */
-    String getName() {
-        return nameObj ? super.getName() : null
-    }
-
-    @Override
-    FileOutParam setPathQualifier(boolean flag) {
-        pathQualifier = flag
-        separatorChar = null
-        return this
-    }
-
-    @Override
-    boolean isPathQualifier() { pathQualifier }
-
-    @Override
-    FileOutParam setOptions(Map opts) {
-        (FileOutParam)super.setOptions(opts)
-    }
-
-}
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/InParam.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/InParam.groovy
deleted file mode 100644
index d41e3c5c28..0000000000
--- a/modules/nextflow/src/main/groovy/nextflow/script/params/InParam.groovy
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Copyright 2013-2023, Seqera Labs
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package nextflow.script.params
-
-import groovyx.gpars.dataflow.DataflowReadChannel
-
-/**
- * Basic interface for *all* input parameters
- *
- * @author Paolo Di Tommaso 
- */
-interface InParam extends Cloneable {
-
-    String getName()
-
-    DataflowReadChannel getInChannel()
-
-    Object getRawChannel()
-
-    short index
-
-    short mapIndex
-
-    def decodeInputs( List values )
-
-}
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/OptionalParam.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/OptionalParam.groovy
deleted file mode 100644
index 326483063f..0000000000
--- a/modules/nextflow/src/main/groovy/nextflow/script/params/OptionalParam.groovy
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Copyright 2013-2023, Seqera Labs
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package nextflow.script.params
-
-
-/**
- * Implements an optional file output option
- *
- * @author Paolo Di Tommaso 
- */
-trait OptionalParam {
-
-    boolean optional
-
-    boolean getOptional() { optional }
-
-    def optional( boolean value ) {
-        this.optional = value
-        return this
-    }
-
-}
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/OutParam.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/OutParam.groovy
deleted file mode 100644
index 3f4977974d..0000000000
--- a/modules/nextflow/src/main/groovy/nextflow/script/params/OutParam.groovy
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Copyright 2013-2023, Seqera Labs
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package nextflow.script.params
-
-import groovyx.gpars.dataflow.DataflowWriteChannel
-
-/**
- * Model a process generic input parameter
- *
- * @author Paolo Di Tommaso 
- */
-
-interface OutParam extends Cloneable {
-
-    /**
-     * @return The parameter name getter
-     */
-    String getName()
-
-    /**
-     * @return The output channel instance
-     */
-    DataflowWriteChannel getOutChannel()
-
-    short getIndex()
-
-    String getChannelEmitName()
-
-}
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/PathQualifier.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/PathQualifier.groovy
deleted file mode 100644
index c9d57b0047..0000000000
--- a/modules/nextflow/src/main/groovy/nextflow/script/params/PathQualifier.groovy
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Copyright 2013-2023, Seqera Labs
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package nextflow.script.params
-
-/**
- * Path qualifier marker interface
- * 
- * @author Paolo Di Tommaso 
- */
-interface PathQualifier {
-
-    def setPathQualifier(boolean flag)
-
-    boolean isPathQualifier()
-
-}
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/StdInParam.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/StdInParam.groovy
deleted file mode 100644
index c465c1ba46..0000000000
--- a/modules/nextflow/src/main/groovy/nextflow/script/params/StdInParam.groovy
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Copyright 2013-2023, Seqera Labs
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package nextflow.script.params
-
-import groovy.transform.InheritConstructors
-import groovy.transform.ToString
-
-
-/**
- * Represents a process *stdin* input parameter
- *
- * @author Paolo Di Tommaso 
- */
-@InheritConstructors
-@ToString(includePackage=false, includeSuper = true)
-class StdInParam extends BaseInParam {
-
-    String getName() { '-' }
-
-    @Override
-    String getTypeName() { 'stdin' }
-
-}
-
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/StdOutParam.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/StdOutParam.groovy
deleted file mode 100644
index cb76414c00..0000000000
--- a/modules/nextflow/src/main/groovy/nextflow/script/params/StdOutParam.groovy
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Copyright 2013-2023, Seqera Labs
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package nextflow.script.params
-
-import groovy.transform.InheritConstructors
-
-/**
- * Model the process *stdout* parameter
- *
- * @author Paolo Di Tommaso 
- */
-@InheritConstructors
-class StdOutParam extends BaseOutParam { }
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/TupleInParam.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/TupleInParam.groovy
deleted file mode 100644
index 6255954eec..0000000000
--- a/modules/nextflow/src/main/groovy/nextflow/script/params/TupleInParam.groovy
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Copyright 2013-2023, Seqera Labs
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package nextflow.script.params
-
-import groovy.transform.InheritConstructors
-import nextflow.NF
-import nextflow.script.TokenEnvCall
-import nextflow.script.TokenFileCall
-import nextflow.script.TokenPathCall
-import nextflow.script.TokenStdinCall
-import nextflow.script.TokenValCall
-import nextflow.script.TokenVar
-
-/**
- * Models a tuple of input parameters
- * 
- * @author Paolo Di Tommaso 
- */
-@InheritConstructors
-class TupleInParam extends BaseInParam {
-
-    protected List inner = []
-
-    @Override String getTypeName() { 'tuple' }
-
-    List getInner() { inner }
-
-    @Override
-    TupleInParam clone() {
-        final copy = (TupleInParam)super.clone()
-        copy.@inner = new ArrayList<>(inner.size())
-        for( BaseInParam p : inner ) {
-            copy.@inner.add((BaseInParam)p.clone())
-        }
-        return copy
-    }
-
-    String getName() { '__$'+this.toString() }
-
-    TupleInParam bind(Object... obj ) {
-
-        for( def item : obj ) {
-
-            if( item instanceof TokenVar ) {
-                throw new IllegalArgumentException("Unqualified input value declaration is not allowed - replace `tuple ${item.name},..` with `tuple val(${item.name}),..`")
-            }
-            else if( item instanceof TokenFileCall ) {
-                newItem(FileInParam).bind( item.target )
-            }
-            else if( item instanceof TokenPathCall ) {
-                newItem(FileInParam)
-                        .setPathQualifier(true)
-                        .setOptions(item.opts)
-                        .bind( item.target )
-            }
-            else if( item instanceof Map ) {
-                throw new IllegalArgumentException("Unqualified input file declaration is not allowed - replace `tuple $item,..` with `tuple path(${item.key}, stageAs:'${item.value}'),..`")
-            }
-            else if( item instanceof TokenValCall ) {
-                newItem(ValueInParam).bind(item.val)
-            }
-            else if( item instanceof TokenEnvCall ) {
-                newItem(EnvInParam).bind(item.val)
-            }
-            else if( item instanceof TokenStdinCall ) {
-                newItem(StdInParam)
-            }
-            else if( item instanceof GString ) {
-                throw new IllegalArgumentException("Unqualified input file declaration is not allowed - replace `tuple \"$item\".. with `tuple path(\"$item\")..`")
-            }
-            else if( item == '-' ) {
-                newItem(StdInParam)
-            }
-            else if( item instanceof String ) {
-                throw new IllegalArgumentException("Unqualified input file declaration is not allowed - replace `tuple '$item',..` with `tuple path('$item'),..`")
-            }
-            else
-                throw new IllegalArgumentException()
-        }
-
-        return this
-
-    }
-
-    private  T newItem( Class type )  {
-        type.newInstance(binding, inner, index)
-    }
-
-}
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/TupleOutParam.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/TupleOutParam.groovy
deleted file mode 100644
index e0de9e24ed..0000000000
--- a/modules/nextflow/src/main/groovy/nextflow/script/params/TupleOutParam.groovy
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Copyright 2013-2023, Seqera Labs
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package nextflow.script.params
-
-import groovy.transform.InheritConstructors
-import nextflow.NF
-import nextflow.script.TokenEnvCall
-import nextflow.script.TokenFileCall
-import nextflow.script.TokenPathCall
-import nextflow.script.TokenStdoutCall
-import nextflow.script.TokenValCall
-import nextflow.script.TokenVar
-/**
- * Model a set of process output parameters
- *
- * @author Paolo Di Tommaso 
- */
-@InheritConstructors
-class TupleOutParam extends BaseOutParam implements OptionalParam {
-
-    protected List inner = new ArrayList<>(10)
-
-    String getName() { toString() }
-
-    List getInner() { inner }
-
-    TupleOutParam clone() {
-        final copy = (TupleOutParam)super.clone()
-        copy.inner = new ArrayList<>(10)
-        for( BaseOutParam p : inner ) {
-            copy.inner.add(p.clone())
-        }
-        return copy
-    }
-
-    TupleOutParam bind(Object... obj ) {
-
-        for( def item : obj ) {
-            if( item instanceof TokenVar ) {
-                throw new IllegalArgumentException("Unqualified output value declaration is not allowed - replace `tuple ${item.name},..` with `tuple val(${item.name}),..`")
-            }
-            else if( item instanceof TokenValCall ) {
-                create(ValueOutParam).bind(item.val)
-            }
-            else if( item instanceof TokenEnvCall ) {
-                create(EnvOutParam).bind(item.val)
-            }
-            else if( item instanceof GString ) {
-                throw new IllegalArgumentException("Unqualified output path declaration is not allowed - replace `tuple \"$item\",..` with `tuple path(\"$item\"),..`")
-            }
-            else if( item instanceof TokenStdoutCall || item == '-'  ) {
-                create(StdOutParam).bind('-')
-            }
-            else if( item instanceof String ) {
-                throw new IllegalArgumentException("Unqualified output path declaration is not allowed - replace `tuple '$item',..` with `tuple path('$item'),..`")
-            }
-            else if( item instanceof TokenFileCall ) {
-                // note that 'filePattern' can be a string or a GString
-                create(FileOutParam).bind(item.target)
-            }
-            else if( item instanceof TokenPathCall ) {
-                // note that 'filePattern' can be a string or a GString
-                create(FileOutParam)
-                        .setPathQualifier(true)
-                        .setOptions(item.opts)
-                        .bind(item.target)
-            }
-            else
-                throw new IllegalArgumentException("Invalid `tuple` output parameter declaration -- item: ${item}")
-        }
-
-        return this
-    }
-
-    protected  T create(Class type) {
-        type.newInstance(binding,inner,index)
-    }
-
-    @Override
-    void lazyInit() {
-        super.lazyInit()
-        inner.each { opt ->
-            if( opt instanceof FileOutParam ) opt.optional(this.optional)
-        }
-    }
-
-}
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/ValueInParam.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/ValueInParam.groovy
deleted file mode 100644
index a4c53d8f9c..0000000000
--- a/modules/nextflow/src/main/groovy/nextflow/script/params/ValueInParam.groovy
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Copyright 2013-2023, Seqera Labs
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package nextflow.script.params
-
-import groovy.transform.InheritConstructors
-
-
-/**
- * Represents a process *value* input parameter
- *
- * @author Paolo Di Tommaso 
- */
-@InheritConstructors
-class ValueInParam extends BaseInParam {
-
-    @Override
-    String getTypeName() { 'val' }
-
-}
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/ValueOutParam.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/ValueOutParam.groovy
deleted file mode 100644
index 81b5a79b3b..0000000000
--- a/modules/nextflow/src/main/groovy/nextflow/script/params/ValueOutParam.groovy
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Copyright 2013-2023, Seqera Labs
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package nextflow.script.params
-
-import groovy.transform.InheritConstructors
-import nextflow.script.TokenVar
-
-
-/**
- * Model a process *value* output parameter
- *
- * @author Paolo Di Tommaso 
- */
-@InheritConstructors
-class ValueOutParam extends BaseOutParam {
-
-    protected target
-
-    String getName() {
-        return nameObj ? super.getName() : null
-    }
-
-    BaseOutParam bind( def obj ) {
-        // the target value object
-        target = obj
-
-        // retrieve the variable name to be used to fetch the value
-        if( obj instanceof TokenVar ) {
-            this.nameObj = obj.name
-        }
-
-        return this
-    }
-
-    /**
-     * Given the {@link nextflow.processor.TaskContext} object resolve the actual value
-     * to which this param is bound
-     *
-     * @param context An instance of {@link nextflow.processor.TaskContext} holding the task evaluation context
-     * @return The actual value to which this out param is bound
-     */
-    def resolve( Map context ) {
-
-        switch( target ) {
-            case TokenVar:
-                return context.get(target.name)
-
-            case Closure:
-                return target.cloneWith(context).call()
-
-            case GString:
-                return target.cloneAsLazy(context).toString()
-
-            default:
-                return target
-        }
-    }
-
-}
diff --git a/modules/nextflow/src/main/groovy/nextflow/util/LazyHelper.groovy b/modules/nextflow/src/main/groovy/nextflow/util/LazyHelper.groovy
new file mode 100644
index 0000000000..6c6e0ad7a8
--- /dev/null
+++ b/modules/nextflow/src/main/groovy/nextflow/util/LazyHelper.groovy
@@ -0,0 +1,285 @@
+/*
+ * Copyright 2013-2023, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package nextflow.util
+
+import groovy.transform.CompileStatic
+import groovy.transform.EqualsAndHashCode
+import groovy.transform.ToString
+/**
+ * Helper methods for lazy binding and resolution.
+ *
+ * @author Paolo Di Tommaso 
+ * @author Ben Sherman 
+ */
+@CompileStatic
+class LazyHelper {
+
+    /**
+     * Resolve a lazy value against a given binding.
+     *
+     * @param binding
+     * @param value
+     */
+    static Object resolve(Object binding, Object value) {
+        if( value instanceof LazyAware )
+            return value.resolve(binding)
+
+        if( value instanceof Closure )
+            return value.cloneWith(binding).call()
+
+        if( value instanceof GString )
+            return value.cloneAsLazy(binding).toString()
+
+        return value
+    }
+
+}
+
+/**
+ * Interface for types that can be lazily resolved
+ */
+interface LazyAware {
+    Object resolve(Object binding)
+}
+
+/**
+ * A list that can be lazily resolved
+ */
+@CompileStatic
+class LazyList implements LazyAware, List {
+
+    @Delegate
+    private List target
+
+    LazyList() {
+        target = []
+    }
+
+    LazyList(int size) {
+        target = new ArrayList(size)
+    }
+
+    LazyList(Collection items) {
+        target = new ArrayList(items)
+    }
+
+    @Override
+    Object resolve(Object binding) {
+        final result = new ArrayList(target.size())
+        for( def item : target )
+            result.add(LazyHelper.resolve(binding, item))
+        return result
+    }
+
+}
+
+/**
+ * A map whose values can be lazily resolved
+ */
+@CompileStatic
+class LazyMap implements Map {
+
+    /** The target map holding the values */
+    @Delegate
+    private Map target
+
+    /** The context map against which dynamic properties are resolved */
+    private Map binding
+
+    private boolean dynamic
+
+    boolean isDynamic() { dynamic }
+
+    protected void setDynamic(boolean val) { dynamic = val }
+
+    protected Map getBinding() { binding }
+
+    void setBinding(Map map) { this.binding = map }
+
+    protected Map getTarget() { target }
+
+    protected void setTarget(Map obj) { this.target = obj }
+
+    LazyMap() {
+        target = new HashMap<>()
+    }
+
+    LazyMap( Map entries ) {
+        assert entries != null
+        target = new HashMap<>()
+        putAll(entries)
+    }
+
+    /**
+     * Resolve a directive *dynamic* value i.e. defined with a closure or lazy string
+     *
+     * @param name The directive name
+     * @param value The value to be resolved
+     * @return The resolved value
+     */
+    protected resolve( String name, value ) {
+
+        /*
+         * directive with one value and optional named parameter are converted
+         * to a list object in which the first element is a map holding the named parameters
+         * and the second is the directive value
+         */
+        if( value instanceof LazyList ) {
+            def copy = new ArrayList(value.size())
+            for( Object item : value ) {
+                if( item instanceof Map )
+                    copy.add( resolveParams(name, item as Map) )
+                else
+                    copy.add( resolveImpl(name, item) )
+            }
+            return copy
+        }
+
+        /*
+         * resolve the values in a map object, preserving
+         * lazy maps as they are
+         */
+        else if( value instanceof Map && value !instanceof LazyMap ) {
+            return resolveParams(name, value)
+        }
+
+        /*
+         * simple value
+         */
+        else {
+            return resolveImpl(name, value)
+        }
+
+    }
+
+    /**
+     * Resolve directive *dynamic* named params
+     *
+     * @param name The directive name
+     * @param value The map holding the named params
+     * @return A map in which dynamic params are resolved to the actual value
+     */
+    private Map resolveParams( String name, Map value ) {
+
+        final copy = new LinkedHashMap()
+        final attr = (value as Map)
+        for( Entry entry : attr.entrySet() ) {
+            copy[entry.key] = resolveImpl(name, entry.value, true)
+        }
+        return copy
+    }
+
+    /**
+     * Resolve a directive dynamic value
+     *
+     * @param name The directive name
+     * @param value The value to be resolved
+     * @param param When {@code true} points that it is a named parameter value, thus closure are only cloned
+     * @return The resolved directive value
+     */
+    private resolveImpl( String name, value, boolean param=false ) {
+
+        if( value instanceof LazyVar ) {
+            return binding.get(value.name)
+        }
+
+        else if( value instanceof Closure ) {
+            def copy = value.cloneWith(getBinding())
+            if( param ) {
+                return copy
+            }
+
+            try {
+                return copy.call()
+            }
+            catch( MissingPropertyException e ) {
+                if( getBinding() == null ) throw new IllegalStateException("Directive `$name` doesn't support dynamic value (or context not yet initialized)")
+                else throw e
+            }
+        }
+
+        else if( value instanceof GString ) {
+            return value.cloneAsLazy(getBinding()).toString()
+        }
+
+        return value
+    }
+
+    /**
+     * Override the get method in such a way that {@link Closure} values are resolved against
+     * the {@link #binding} map
+     *
+     * @param key The map entry key
+     * @return The associated value
+     */
+    Object get( key ) {
+        return getValue(key)
+    }
+
+    Object getValue(Object key) {
+        final value = target.get(key)
+        return resolve(key as String, value)
+    }
+
+    Object put( String key, Object value ) {
+        if( value instanceof Closure ) {
+            dynamic |= true
+        }
+        else if( value instanceof GString ) {
+            for( int i=0; i put(k as String, v) }
+    }
+
+    @Override
+    String toString() {
+        final allKeys = keySet()
+        final result = new ArrayList(allKeys.size())
+        for( String key : allKeys ) { result << "$key: ${getProperty(key)}".toString() }
+        result.join('; ')
+    }
+
+}
+
+/**
+ * A variable that can be lazily resolved
+ */
+@CompileStatic
+@EqualsAndHashCode
+@ToString
+class LazyVar implements LazyAware {
+    String name
+
+    LazyVar(String name) {
+        this.name = name
+    }
+
+    @Override
+    Object resolve(Object binding) {
+        if( binding !instanceof Map )
+            throw new IllegalArgumentException("Can't resolve lazy var `$name` because the given binding is not a map")
+
+        return ((Map)binding).get(name)
+    }
+}

From bca231b621537d89eabd209ee4f6161cf1e12ba9 Mon Sep 17 00:00:00 2001
From: Ben Sherman 
Date: Fri, 8 Dec 2023 23:18:41 -0600
Subject: [PATCH 15/36] Move ProcessBuilder#applyConfig() into subclass

Signed-off-by: Ben Sherman 
---
 .../src/main/groovy/nextflow/Session.groovy   |   4 +-
 .../groovy/nextflow/script/ProcessDef.groovy  |   4 +-
 .../nextflow/script/ProcessFactory.groovy     |   6 +-
 .../nextflow/script/dsl/ProcessBuilder.groovy | 196 ---------------
 .../script/dsl/ProcessConfigBuilder.groovy    | 231 ++++++++++++++++++
 5 files changed, 238 insertions(+), 203 deletions(-)
 create mode 100644 modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessConfigBuilder.groovy

diff --git a/modules/nextflow/src/main/groovy/nextflow/Session.groovy b/modules/nextflow/src/main/groovy/nextflow/Session.groovy
index 9572b1c578..2c426d672a 100644
--- a/modules/nextflow/src/main/groovy/nextflow/Session.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/Session.groovy
@@ -55,7 +55,7 @@ import nextflow.processor.ErrorStrategy
 import nextflow.processor.TaskFault
 import nextflow.processor.TaskHandler
 import nextflow.processor.TaskProcessor
-import nextflow.script.dsl.ProcessBuilder
+import nextflow.script.dsl.ProcessConfigBuilder
 import nextflow.script.BaseScript
 import nextflow.script.ProcessConfig
 import nextflow.script.ProcessFactory
@@ -932,7 +932,7 @@ class Session implements ISession {
      * @return {@code true} if the name specified belongs to the list of process names or {@code false} otherwise
      */
     protected boolean checkValidProcessName(Collection processNames, String selector, List errorMessage)  {
-        final matches = processNames.any { name -> ProcessBuilder.matchesSelector(name, selector) }
+        final matches = processNames.any { name -> ProcessConfigBuilder.matchesSelector(name, selector) }
         if( matches )
             return true
 
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy
index e74fc2dd40..a23a822a16 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy
@@ -25,7 +25,7 @@ import nextflow.Session
 import nextflow.exception.ScriptRuntimeException
 import nextflow.extension.CH
 import nextflow.extension.CombineOp
-import nextflow.script.dsl.ProcessBuilder
+import nextflow.script.dsl.ProcessConfigBuilder
 
 /**
  * Models a nextflow process definition
@@ -90,7 +90,7 @@ class ProcessDef extends BindableDef implements IterableDef, ChainableDef {
 
     protected void initialize() {
         // apply config settings to the process
-        new ProcessBuilder(config).applyConfig((Map)session.config.process, baseName, simpleName, processName)
+        new ProcessConfigBuilder(config).applyConfig((Map)session.config.process, baseName, simpleName, processName)
     }
 
     @Override
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessFactory.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessFactory.groovy
index 3d660c8774..e05cfd5dcf 100755
--- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessFactory.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessFactory.groovy
@@ -23,6 +23,7 @@ import nextflow.Session
 import nextflow.executor.Executor
 import nextflow.executor.ExecutorFactory
 import nextflow.processor.TaskProcessor
+import nextflow.script.dsl.ProcessConfigBuilder
 import nextflow.script.dsl.ProcessDsl
 /**
  *  Factory class for {@TaskProcessor} instances
@@ -96,11 +97,10 @@ class ProcessFactory {
             throw new IllegalArgumentException("Missing script in the specified process block -- make sure it terminates with the script string to be executed")
 
         // -- apply settings from config file to process config
-        builder.applyConfig((Map)config.process, name, null, null)
-
-        // -- the config object
         final processConfig = builder.getConfig()
 
+        new ProcessConfigBuilder(processConfig).applyConfig((Map)config.process, name, null, null)
+
         // -- get the executor for the given process config
         final execObj = executorFactory.getExecutor(name, processConfig, script, session)
 
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessBuilder.groovy b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessBuilder.groovy
index 4db834b630..6266892d0b 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessBuilder.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessBuilder.groovy
@@ -20,7 +20,6 @@ import java.util.regex.Pattern
 
 import groovy.util.logging.Slf4j
 import nextflow.ast.DslCodeVisitor
-import nextflow.exception.ConfigParseException
 import nextflow.exception.IllegalConfigException
 import nextflow.exception.IllegalDirectiveException
 import nextflow.exception.ScriptRuntimeException
@@ -449,199 +448,4 @@ class ProcessBuilder {
         return new ProcessDef(ownerScript, processName, body, config)
     }
 
-    /// CONFIG
-
-    /**
-     * Apply process config settings from the config file to a process.
-     *
-     * @param configDirectives
-     * @param baseName
-     * @param simpleName
-     * @param fullyQualifiedName
-     */
-    void applyConfig(Map configDirectives, String baseName, String simpleName, String fullyQualifiedName) {
-        // -- apply settings defined in the config object using the`withLabel:` syntax
-        final processLabels = config.getLabels() ?: ['']
-        applyConfigSelectorWithLabels(configDirectives, processLabels)
-
-        // -- apply settings defined in the config file using the process base name
-        applyConfigSelectorWithName(configDirectives, baseName)
-
-        // -- apply settings defined in the config file using the process simple name
-        if( simpleName && simpleName!=baseName )
-            applyConfigSelectorWithName(configDirectives, simpleName)
-
-        // -- apply settings defined in the config file using the process fully qualified name (ie. with the execution scope)
-        if( fullyQualifiedName && (fullyQualifiedName!=simpleName || fullyQualifiedName!=baseName) )
-            applyConfigSelectorWithName(configDirectives, fullyQualifiedName)
-
-        // -- apply defaults
-        applyConfigDefaults(configDirectives)
-
-        // -- check for conflicting settings
-        if( config.scratch && config.stageInMode == 'rellink' ) {
-            log.warn("Directives `scratch` and `stageInMode=rellink` conflict with each other -- Enforcing default stageInMode for process `$simpleName`")
-            config.remove('stageInMode')
-        }
-    }
-
-    /**
-     * Apply the config settings in a label selector, for example:
-     *
-     * ```
-     * process {
-     *     withLabel: foo {
-     *         cpus = 1
-     *         memory = 2.gb
-     *     }
-     * }
-     * ```
-     *
-     * @param configDirectives
-     * @param labels
-     */
-    protected void applyConfigSelectorWithLabels(Map configDirectives, List labels) {
-        final prefix = 'withLabel:'
-        for( String rule : configDirectives.keySet() ) {
-            if( !rule.startsWith(prefix) )
-                continue
-            final pattern = rule.substring(prefix.size()).trim()
-            if( !matchesLabels(labels, pattern) )
-                continue
-
-            log.debug "Config settings `$rule` matches labels `${labels.join(',')}` for process with name $processName"
-            final settings = configDirectives.get(rule)
-            if( settings instanceof Map ) {
-                applyConfigSettings(settings)
-            }
-            else if( settings != null ) {
-                throw new ConfigParseException("Unknown config settings for process labeled ${labels.join(',')} -- settings=$settings ")
-            }
-        }
-    }
-
-    static boolean matchesLabels(List labels, String pattern) {
-        final isNegated = pattern.startsWith('!')
-        if( isNegated )
-            pattern = pattern.substring(1).trim()
-
-        final regex = Pattern.compile(pattern)
-        for (label in labels) {
-            if (regex.matcher(label).matches()) {
-                return !isNegated
-            }
-        }
-
-        return isNegated
-    }
-
-    /**
-     * Apply the config settings in a name selector, for example:
-     *
-     * ```
-     * process {
-     *     withName: foo {
-     *         cpus = 1
-     *         memory = 2.gb
-     *     }
-     * }
-     * ```
-     *
-     * @param configDirectives
-     * @param target
-     */
-    protected void applyConfigSelectorWithName(Map configDirectives, String target) {
-        final prefix = 'withName:'
-        for( String rule : configDirectives.keySet() ) {
-            if( !rule.startsWith(prefix) )
-                continue
-            final pattern = rule.substring(prefix.size()).trim()
-            if( !matchesSelector(target, pattern) )
-                continue
-
-            log.debug "Config settings `$rule` matches process $processName"
-            def settings = configDirectives.get(rule)
-            if( settings instanceof Map ) {
-                applyConfigSettings(settings)
-            }
-            else if( settings != null ) {
-                throw new ConfigParseException("Unknown config settings for process with name: $target  -- settings=$settings ")
-            }
-        }
-    }
-
-    static boolean matchesSelector(String name, String pattern) {
-        final isNegated = pattern.startsWith('!')
-        if( isNegated )
-            pattern = pattern.substring(1).trim()
-        return Pattern.compile(pattern).matcher(name).matches() ^ isNegated
-    }
-
-
-    /**
-     * Apply config settings to a process.
-     *
-     * @param settings
-     */
-    protected void applyConfigSettings(Map settings) {
-        if( !settings )
-            return
-
-        for( def entry : settings ) {
-            if( entry.key.startsWith("withLabel:") || entry.key.startsWith("withName:"))
-                continue
-
-            if( !DIRECTIVES.contains(entry.key) )
-                log.warn "Unknown directive `$entry.key` for process `$processName`"
-
-            if( entry.key == 'params' ) // <-- patch issue #242
-                continue
-
-            if( entry.key == 'ext' ) {
-                if( config.getProperty('ext') instanceof Map ) {
-                    // update missing 'ext' properties found in 'process' scope
-                    def ext = config.getProperty('ext') as Map
-                    entry.value.each { String k, v -> ext[k] = v }
-                }
-                continue
-            }
-
-            putWithRepeat(entry.key, entry.value)
-        }
-    }
-
-    /**
-     * Apply the global settings in the process config scope to a process.
-     *
-     * @param defaults
-     */
-    protected void applyConfigDefaults( Map defaults ) {
-        for( String key : defaults.keySet() ) {
-            if( key == 'params' )
-                continue
-            final value = defaults.get(key)
-            final current = config.getProperty(key)
-            if( key == 'ext' ) {
-                if( value instanceof Map && current instanceof Map ) {
-                    final ext = current as Map
-                    value.each { k,v -> if(!ext.containsKey(k)) ext.put(k,v) }
-                }
-            }
-            else if( !config.containsKey(key) || (ProcessConfig.DEFAULT_CONFIG.containsKey(key) && current==ProcessConfig.DEFAULT_CONFIG.get(key)) ) {
-                putWithRepeat(key, value)
-            }
-        }
-    }
-
-    private static final List REPEATABLE_DIRECTIVES = ['label','module','pod','publishDir']
-
-    protected void putWithRepeat( String name, Object value ) {
-        if( name in REPEATABLE_DIRECTIVES ) {
-            config.remove(name)
-            this.metaClass.invokeMethod(this, name, value)
-        }
-        else {
-            config.put(name, value)
-        }
-    }
 }
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessConfigBuilder.groovy b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessConfigBuilder.groovy
new file mode 100644
index 0000000000..4559e6492c
--- /dev/null
+++ b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessConfigBuilder.groovy
@@ -0,0 +1,231 @@
+/*
+ * Copyright 2013-2023, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package nextflow.script.dsl
+
+import java.util.regex.Pattern
+
+import groovy.util.logging.Slf4j
+import nextflow.exception.ConfigParseException
+import nextflow.script.ProcessConfig
+
+/**
+ * Builder for {@link ProcessConfig}.
+ *
+ * @author Ben Sherman 
+ */
+@Slf4j
+class ProcessConfigBuilder extends ProcessBuilder {
+
+    ProcessConfigBuilder(ProcessConfig config) {
+        super(config)
+    }
+
+    /**
+     * Apply process config settings from the config file to a process.
+     *
+     * @param configDirectives
+     * @param baseName
+     * @param simpleName
+     * @param fullyQualifiedName
+     */
+    void applyConfig(Map configDirectives, String baseName, String simpleName, String fullyQualifiedName) {
+        // -- apply settings defined in the config object using the`withLabel:` syntax
+        final processLabels = config.getLabels() ?: ['']
+        applyConfigSelectorWithLabels(configDirectives, processLabels)
+
+        // -- apply settings defined in the config file using the process base name
+        applyConfigSelectorWithName(configDirectives, baseName)
+
+        // -- apply settings defined in the config file using the process simple name
+        if( simpleName && simpleName!=baseName )
+            applyConfigSelectorWithName(configDirectives, simpleName)
+
+        // -- apply settings defined in the config file using the process fully qualified name (ie. with the execution scope)
+        if( fullyQualifiedName && (fullyQualifiedName!=simpleName || fullyQualifiedName!=baseName) )
+            applyConfigSelectorWithName(configDirectives, fullyQualifiedName)
+
+        // -- apply defaults
+        applyConfigDefaults(configDirectives)
+
+        // -- check for conflicting settings
+        if( config.scratch && config.stageInMode == 'rellink' ) {
+            log.warn("Directives `scratch` and `stageInMode=rellink` conflict with each other -- Enforcing default stageInMode for process `$simpleName`")
+            config.remove('stageInMode')
+        }
+    }
+
+    /**
+     * Apply the config settings in a label selector, for example:
+     *
+     * ```
+     * process {
+     *     withLabel: foo {
+     *         cpus = 1
+     *         memory = 2.gb
+     *     }
+     * }
+     * ```
+     *
+     * @param configDirectives
+     * @param labels
+     */
+    protected void applyConfigSelectorWithLabels(Map configDirectives, List labels) {
+        final prefix = 'withLabel:'
+        for( String rule : configDirectives.keySet() ) {
+            if( !rule.startsWith(prefix) )
+                continue
+            final pattern = rule.substring(prefix.size()).trim()
+            if( !matchesLabels(labels, pattern) )
+                continue
+
+            log.debug "Config settings `$rule` matches labels `${labels.join(',')}` for process with name $processName"
+            final settings = configDirectives.get(rule)
+            if( settings instanceof Map ) {
+                applyConfigSettings(settings)
+            }
+            else if( settings != null ) {
+                throw new ConfigParseException("Unknown config settings for process labeled ${labels.join(',')} -- settings=$settings ")
+            }
+        }
+    }
+
+    static boolean matchesLabels(List labels, String pattern) {
+        final isNegated = pattern.startsWith('!')
+        if( isNegated )
+            pattern = pattern.substring(1).trim()
+
+        final regex = Pattern.compile(pattern)
+        for (label in labels) {
+            if (regex.matcher(label).matches()) {
+                return !isNegated
+            }
+        }
+
+        return isNegated
+    }
+
+    /**
+     * Apply the config settings in a name selector, for example:
+     *
+     * ```
+     * process {
+     *     withName: foo {
+     *         cpus = 1
+     *         memory = 2.gb
+     *     }
+     * }
+     * ```
+     *
+     * @param configDirectives
+     * @param target
+     */
+    protected void applyConfigSelectorWithName(Map configDirectives, String target) {
+        final prefix = 'withName:'
+        for( String rule : configDirectives.keySet() ) {
+            if( !rule.startsWith(prefix) )
+                continue
+            final pattern = rule.substring(prefix.size()).trim()
+            if( !matchesSelector(target, pattern) )
+                continue
+
+            log.debug "Config settings `$rule` matches process $processName"
+            def settings = configDirectives.get(rule)
+            if( settings instanceof Map ) {
+                applyConfigSettings(settings)
+            }
+            else if( settings != null ) {
+                throw new ConfigParseException("Unknown config settings for process with name: $target  -- settings=$settings ")
+            }
+        }
+    }
+
+    static boolean matchesSelector(String name, String pattern) {
+        final isNegated = pattern.startsWith('!')
+        if( isNegated )
+            pattern = pattern.substring(1).trim()
+        return Pattern.compile(pattern).matcher(name).matches() ^ isNegated
+    }
+
+
+    /**
+     * Apply config settings to a process.
+     *
+     * @param settings
+     */
+    protected void applyConfigSettings(Map settings) {
+        if( !settings )
+            return
+
+        for( def entry : settings ) {
+            if( entry.key.startsWith("withLabel:") || entry.key.startsWith("withName:"))
+                continue
+
+            if( !DIRECTIVES.contains(entry.key) )
+                log.warn "Unknown directive `$entry.key` for process `$processName`"
+
+            if( entry.key == 'params' ) // <-- patch issue #242
+                continue
+
+            if( entry.key == 'ext' ) {
+                if( config.getProperty('ext') instanceof Map ) {
+                    // update missing 'ext' properties found in 'process' scope
+                    def ext = config.getProperty('ext') as Map
+                    entry.value.each { String k, v -> ext[k] = v }
+                }
+                continue
+            }
+
+            putWithRepeat(entry.key, entry.value)
+        }
+    }
+
+    /**
+     * Apply the global settings in the process config scope to a process.
+     *
+     * @param defaults
+     */
+    protected void applyConfigDefaults( Map defaults ) {
+        for( String key : defaults.keySet() ) {
+            if( key == 'params' )
+                continue
+            final value = defaults.get(key)
+            final current = config.getProperty(key)
+            if( key == 'ext' ) {
+                if( value instanceof Map && current instanceof Map ) {
+                    final ext = current as Map
+                    value.each { k,v -> if(!ext.containsKey(k)) ext.put(k,v) }
+                }
+            }
+            else if( !config.containsKey(key) || (ProcessConfig.DEFAULT_CONFIG.containsKey(key) && current==ProcessConfig.DEFAULT_CONFIG.get(key)) ) {
+                putWithRepeat(key, value)
+            }
+        }
+    }
+
+    private static final List REPEATABLE_DIRECTIVES = ['label','module','pod','publishDir']
+
+    protected void putWithRepeat( String name, Object value ) {
+        if( name in REPEATABLE_DIRECTIVES ) {
+            config.remove(name)
+            this.metaClass.invokeMethod(this, name, value)
+        }
+        else {
+            config.put(name, value)
+        }
+    }
+
+}

From 872a3e208ca72271d6b1aca867a2e5c8b8305342 Mon Sep 17 00:00:00 2001
From: Ben Sherman 
Date: Sat, 9 Dec 2023 09:34:37 -0600
Subject: [PATCH 16/36] Add CombineManyOp to combine process input channels

Signed-off-by: Ben Sherman 
---
 docs/operator.md                              |   5 -
 .../nextflow/extension/CombineManyOp.groovy   | 102 ++++++++++++++++++
 .../nextflow/extension/CombineOp.groovy       |  20 ++--
 .../nextflow/extension/DataflowHelper.groovy  |   4 +-
 .../nextflow/extension/OperatorImpl.groovy    |   3 +-
 .../nextflow/processor/TaskProcessor.groovy   |   4 +-
 .../groovy/nextflow/script/ProcessDef.groovy  |  32 +++---
 .../nextflow/script/ProcessInput.groovy       |  34 ++++--
 .../nextflow/script/dsl/ProcessDsl.groovy     |   9 +-
 9 files changed, 153 insertions(+), 60 deletions(-)
 create mode 100644 modules/nextflow/src/main/groovy/nextflow/extension/CombineManyOp.groovy

diff --git a/docs/operator.md b/docs/operator.md
index 028099ecec..a54f0b022e 100644
--- a/docs/operator.md
+++ b/docs/operator.md
@@ -334,11 +334,6 @@ A second version of the `combine` operator allows you to combine items that shar
 :language: console
 ```
 
-:::{versionadded} 24.10.0
-:::
-
-By default, the `combine` operator flattens list items into the resulting tuple. You can set `flat: false` to preserve nested list items.
-
 See also [join](#join) and [cross](#cross).
 
 (operator-concat)=
diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/CombineManyOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/CombineManyOp.groovy
new file mode 100644
index 0000000000..ec2a2a14ab
--- /dev/null
+++ b/modules/nextflow/src/main/groovy/nextflow/extension/CombineManyOp.groovy
@@ -0,0 +1,102 @@
+/*
+ * Copyright 2013-2023, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package nextflow.extension
+
+import java.util.concurrent.atomic.AtomicInteger
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import groovyx.gpars.dataflow.DataflowReadChannel
+import groovyx.gpars.dataflow.DataflowWriteChannel
+import nextflow.Channel
+/**
+ * Operator for combining many source channels into a single channel,
+ * with the option to only merge channels that are not marked as "iterators".
+ *
+ * @see ProcessDef#collectInputs(Object[])
+ *
+ * @author Ben Sherman 
+ */
+@Slf4j
+@CompileStatic
+class CombineManyOp {
+
+    private List sources
+
+    private List iterators
+
+    private boolean singleton
+
+    private List queues = []
+
+    private transient List combinations
+
+    CombineManyOp(List sources, List iterators) {
+        this.sources = sources
+        this.iterators = iterators
+        this.singleton = iterators.size() == 0 && sources.every(ch -> !CH.isChannelQueue(ch))
+        this.queues = sources.collect( it -> [] )
+    }
+
+    private Map handler(int index, DataflowWriteChannel target, AtomicInteger counter) {
+        final opts = new LinkedHashMap(2)
+        opts.onNext = {
+            onNext(target, index, it)
+        }
+        opts.onComplete = {
+            if( counter.decrementAndGet() == 0 && !singleton )
+                target.bind(Channel.STOP)
+        }
+        return opts
+    }
+
+    private synchronized void onNext(DataflowWriteChannel target, int index, Object value) {
+        queues[index].add(value)
+
+        // wait until every source has a value
+        if( queues.any(q -> q.size() == 0) )
+            return
+
+        // emit the next item if there are no iterators
+        if( iterators.size() == 0 ) {
+            final args = queues.collect(q -> q.pop())
+            target.bind(args)
+            return
+        }
+
+        // otherwise emit an item for every iterator combination
+        if( combinations == null )
+            combinations = iterators.collect( i -> queues[i].first() ).combinations()
+
+        final args = (0.. i in iterators ? null : queues[i].pop() )
+        for( List entries : combinations ) {
+            for( int k = 0; k < entries.size(); k++ )
+                args[iterators[k]] = entries[k]
+
+            target.bind(args)
+        }
+    }
+
+    DataflowWriteChannel apply() {
+        final target = CH.create(singleton)
+        final counter = new AtomicInteger(sources.size())
+        for( int i = 0; i < sources.size(); i++ )
+            DataflowHelper.subscribeImpl( sources[i], handler(i, target, counter) )
+
+        return target
+    }
+}
diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/CombineOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/CombineOp.groovy
index c06b2ebd56..729e500004 100644
--- a/modules/nextflow/src/main/groovy/nextflow/extension/CombineOp.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/extension/CombineOp.groovy
@@ -54,9 +54,7 @@ class CombineOp {
 
     private List pivot = NONE
 
-    private boolean flat = true
-
-    CombineOp(DataflowReadChannel left, Object right, Map opts) {
+    CombineOp(DataflowReadChannel left, Object right) {
 
         leftChannel = left
 
@@ -74,11 +72,6 @@ class CombineOp {
                 throw new IllegalArgumentException("Not a valid argument for 'combine' operator [${right?.class?.simpleName}]: ${right} -- Use a List or a channel instead. ")
         }
 
-        if( opts?.by != null )
-            pivot = opts.by as List
-
-        if( opts?.flat != null )
-            flat = opts.flat
     }
 
     CombineOp setPivot( pivot ) {
@@ -110,8 +103,7 @@ class CombineOp {
         opts.onComplete = {
             if( stopCount.decrementAndGet()==0) {
                 target << Channel.STOP
-            }
-        }
+            }}
 
         return opts
     }
@@ -121,8 +113,8 @@ class CombineOp {
     def tuple( List p, a, b ) {
         List result = new LinkedList()
         result.addAll(p)
-        addToList(result, a, flat)
-        addToList(result, b, flat)
+        addToList(result, a)
+        addToList(result, b)
 
         result.size()==1 ? result[0] : result
     }
@@ -151,7 +143,7 @@ class CombineOp {
             return
         }
 
-        throw new IllegalArgumentException("Not a valid combine operator index: $index")
+        throw new IllegalArgumentException("Not a valid spread operator index: $index")
     }
 
     DataflowWriteChannel apply() {
@@ -170,7 +162,7 @@ class CombineOp {
         }
 
         else
-            throw new IllegalArgumentException("Not a valid combine operator state -- Missing right operand")
+            throw new IllegalArgumentException("Not a valid spread operator state -- Missing right operand")
 
         return target
     }
diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy
index bb10a8c868..dd4cdd5b5a 100644
--- a/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy
@@ -381,8 +381,8 @@ class DataflowHelper {
 
     @PackageScope
     @CompileStatic
-    static void addToList(List result, Object entry, boolean flat=true)  {
-        if( flat && entry instanceof List ) {
+    static void addToList(List result, entry)  {
+        if( entry instanceof List ) {
             result.addAll(entry)
         }
         else {
diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy
index e622173307..ceeb26f858 100644
--- a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy
@@ -715,8 +715,9 @@ class OperatorImpl {
     DataflowWriteChannel combine( DataflowReadChannel left, Map params, Object right ) {
         checkParams('combine', params, [flat:Boolean, by: [List,Integer]])
 
-        final op = new CombineOp(left,right,params)
+        final op = new CombineOp(left,right)
         OpCall.current.get().inputs.addAll(op.inputs)
+        if( params?.by != null ) op.pivot = params.by
         final target = op.apply()
         return target
     }
diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy
index a491da6f6a..cc5de94f66 100644
--- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy
@@ -432,9 +432,7 @@ class TaskProcessor {
         // this allows us to manage them independently from the operator life-cycle
         def interceptor = new TaskProcessorInterceptor(source, control, singleton)
         def params = [inputs: opInputs, maxForks: session.poolSize, listeners: [interceptor] ]
-        def invoke = this.&invokeTask
-
-        this.operator = new DataflowOperator(group, params, invoke)
+        this.operator = new DataflowOperator(group, params, this.&invokeTask)
 
         // notify the creation of a new vertex the execution DAG
         NodeMarker.addProcessNode(this, config.getInputs(), config.getOutputs())
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy
index a23a822a16..0cad157f6b 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy
@@ -21,10 +21,11 @@ import groovy.util.logging.Slf4j
 import groovyx.gpars.dataflow.DataflowReadChannel
 import nextflow.Const
 import nextflow.Global
+import nextflow.NF
 import nextflow.Session
 import nextflow.exception.ScriptRuntimeException
 import nextflow.extension.CH
-import nextflow.extension.CombineOp
+import nextflow.extension.CombineManyOp
 import nextflow.script.dsl.ProcessConfigBuilder
 
 /**
@@ -181,32 +182,23 @@ class ProcessDef extends BindableDef implements IterableDef, ChainableDef {
             return source
         }
 
-        // set input channels
+        // create input channels
         for( int i = 0; i < declaredInputs.size(); i++ )
             declaredInputs[i].bind(args[i])
 
-        // normalize args into channels
-        final inputs = declaredInputs.getChannels()
-
-        // make sure no more than one queue channel is provided
-        int count = 0
-        for( int i = 0; i < inputs.size(); i++ )
-            if( CH.isChannelQueue(inputs[i]) && !declaredInputs[i].isIterator() )
-                count += 1
-
-        if( count > 1 )
-            throw new ScriptRuntimeException("Process `$name` received multiple queue channel inputs which is not allowed -- consider combining these channels explicitly using the `combine` or `join` operator")
-
         // combine input channels
-        def result = inputs.first()
-
+        final inputs = declaredInputs.getChannels()
         if( inputs.size() == 1 )
-            return result.chainWith( it -> [it] )
+            return inputs.first().chainWith( it -> [it] )
 
-        for( int i = 1; i < inputs.size(); i++ )
-            result = CH.getReadChannel(new CombineOp(result, inputs[i], [flat: false]).apply())
+        final count = (0.. CH.isChannelQueue(inputs[i]) && !declaredInputs[i].isIterator()
+        )
+        if( NF.isStrictMode() && count > 1 )
+            throw new ScriptRuntimeException("Process `$name` received multiple queue channel inputs which will be implicitly mergeed -- consider combining them explicitly with `combine` or `join`, or converting single-item chennels into value channels with `collect` or `first`")
 
-        return result
+        final iterators = (0.. declaredInputs[i].isIterator() )
+        return CH.getReadChannel(new CombineManyOp(inputs, iterators).apply())
     }
 
     private void collectOutputs(boolean singleton) {
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessInput.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessInput.groovy
index 61c3a35116..5eccb7a294 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessInput.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessInput.groovy
@@ -19,7 +19,10 @@ package nextflow.script
 import groovy.transform.CompileStatic
 import groovyx.gpars.dataflow.DataflowBroadcast
 import groovyx.gpars.dataflow.DataflowReadChannel
+import groovyx.gpars.dataflow.DataflowVariable
+import groovyx.gpars.dataflow.expression.DataflowExpression
 import nextflow.extension.CH
+import nextflow.extension.ToListOp
 
 /**
  * Models a process input.
@@ -57,28 +60,39 @@ class ProcessInput implements Cloneable {
         if( obj == null )
             throw new IllegalArgumentException('A process input channel evaluates to null')
 
-        final value = obj instanceof Closure
+        def value = obj instanceof Closure
             ? obj.call()
             : obj
 
         if( value == null )
             throw new IllegalArgumentException('A process input channel evaluates to null')
 
-        if( iterator ) {
-            final result = CH.create()
-            CH.emitAndClose(result, value instanceof Collection ? value : [value])
-            return CH.getReadChannel(result)
-        }
+        if( iterator )
+            value = getIteratorChannel(value)
 
-        else if( value instanceof DataflowReadChannel || value instanceof DataflowBroadcast ) {
+        if( value instanceof DataflowReadChannel || value instanceof DataflowBroadcast )
             return CH.getReadChannel(value)
-        }
 
+        final result = CH.value()
+        result.bind(value)
+        return result
+    }
+
+    private DataflowReadChannel getIteratorChannel(Object value) {
+        def result
+        if( value instanceof DataflowExpression ) {
+            result = value
+        }
+        else if( CH.isChannel(value) ) {
+            def read = CH.getReadChannel(value)
+            result = new ToListOp(read).apply()
+        }
         else {
-            final result = CH.value()
+            result = new DataflowVariable()
             result.bind(value)
-            return result
         }
+
+        return result.chainWith { it instanceof Collection || it == null ? it : [it] }
     }
 
     DataflowReadChannel getChannel() {
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessDsl.groovy b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessDsl.groovy
index 0bd3381af0..96ed502a3e 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessDsl.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessDsl.groovy
@@ -221,9 +221,8 @@ class ProcessDsl extends ProcessBuilder {
             throw new IllegalArgumentException("Output `tuple` must define at least two elements -- Check process `$processName`")
 
         // separate param options from path options
-        final paramOpts = [optional: opts.optional]
         if( opts.emit )
-            paramOpts.name = opts.remove('emit')
+            opts.name = opts.remove('emit')
 
         // make lazy list with tuple elements
         final target = new LazyList(elements.size())
@@ -244,12 +243,12 @@ class ProcessDsl extends ProcessBuilder {
             }
             else if( item instanceof TokenFileCall ) {
                 // file pattern can be a String or GString
-                final key = _out_path0(item.target, false, [:])
+                final key = _out_path0(item.target, false, [optional: opts.optional])
                 target << new LazyPathCall(key)
             }
             else if( item instanceof TokenPathCall ) {
                 // file pattern can be a String or GString
-                final key = _out_path0(item.target, true, item.opts)
+                final key = _out_path0(item.target, true, item.opts + [optional: opts.optional])
                 target << new LazyPathCall(key)
             }
             else if( item instanceof GString ) {
@@ -265,7 +264,7 @@ class ProcessDsl extends ProcessBuilder {
                 throw new IllegalArgumentException("Invalid `tuple` output parameter declaration -- item: ${item}")
         }
 
-        outputs.addParam(target, paramOpts)
+        outputs.addParam(target, opts)
     }
 
     void _out_val(Map opts=[:], Object target) {

From 72b54f68e8140d6373507a6779c578f3e376680c Mon Sep 17 00:00:00 2001
From: Ben Sherman 
Date: Sat, 9 Dec 2023 13:23:26 -0600
Subject: [PATCH 17/36] Save variable refs in ProcessFn

Signed-off-by: Ben Sherman 
---
 .../src/main/groovy/nextflow/ast/ProcessFn.groovy    |  1 +
 .../main/groovy/nextflow/ast/ProcessFnXform.groovy   | 12 ++++++++++++
 .../main/groovy/nextflow/processor/TaskRun.groovy    |  5 +----
 .../main/groovy/nextflow/script/BaseScript.groovy    |  6 +++++-
 4 files changed, 19 insertions(+), 5 deletions(-)

diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/ProcessFn.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/ProcessFn.groovy
index ca268dc668..11ce865f49 100644
--- a/modules/nextflow/src/main/groovy/nextflow/ast/ProcessFn.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/ast/ProcessFn.groovy
@@ -39,4 +39,5 @@ import java.lang.annotation.Target
     // injected via AST transform
     String[] params()
     String source()
+    String[] vars()
 }
diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/ProcessFnXform.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/ProcessFnXform.groovy
index b872f815f2..67db63a86c 100644
--- a/modules/nextflow/src/main/groovy/nextflow/ast/ProcessFnXform.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/ast/ProcessFnXform.groovy
@@ -94,6 +94,12 @@ class ProcessFnXform extends ClassCodeVisitorSupport {
         // append script source
         annotation.addMember( 'source', constX( getSource(method.getCode()) ) )
 
+        // append variable references so that global vars can be cached
+        final vars = getVariableRefs(method.getCode())
+        annotation.addMember( 'vars', new ListExpression(
+            vars.collect(var -> (Expression)constX(var))
+        ) )
+
         // prepend `task` method parameter
         params.push(new Parameter(new ClassNode(TaskConfig), 'task'))
         method.setParameters(params as Parameter[])
@@ -130,6 +136,12 @@ class ProcessFnXform extends ClassCodeVisitorSupport {
         ) )
     }
 
+    protected List getVariableRefs(Statement stmt) {
+        final visitor = new VariableVisitor(unit)
+        stmt.visit(visitor)
+        return visitor.getAllVariables().collect( ref -> ref.name )
+    }
+
     private String getSource(ASTNode node) {
         final buffer = new StringBuilder()
         final colx = node.getColumnNumber()
diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy
index 6fef194101..a586b31f26 100644
--- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy
@@ -763,10 +763,7 @@ class TaskRun implements Cloneable {
         final result = new HashMap(variableNames.size())
         final processName = name
 
-        def itr = variableNames.iterator()
-        while( itr.hasNext() ) {
-            final varName = itr.next()
-
+        for( def varName : variableNames ) {
             final p = varName.indexOf('.')
             final baseName = p !=- 1 ? varName.substring(0,p) : varName
 
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy b/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy
index 2b69237bae..6bf7fde7c5 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy
@@ -223,10 +223,14 @@ abstract class BaseScript extends Script implements ExecutionContext {
             : processFn.shell() ? 'shell'
             : 'exec'
 
+        // -- variable references
+        final valRefs = processFn.vars().collect( var -> new TokenValRef(var) )
+
+        // -- build process
         final process = builder
             .withInputs(inputs.build())
             .withOutputs(outputs.build())
-            .withBody(this.&"${name}", type, processFn.source())
+            .withBody(this.&"${name}", type, processFn.source(), valRefs)
             .build()
 
         // register process

From dfd5aea4cff76b8bfe07172f65b5bd0fe9ef75a7 Mon Sep 17 00:00:00 2001
From: Ben Sherman 
Date: Sat, 9 Dec 2023 16:49:18 -0600
Subject: [PATCH 18/36] Fix bugs

Signed-off-by: Ben Sherman 
---
 .../nextflow/extension/CombineManyOp.groovy   | 25 +++++++++++++------
 .../nextflow/script/ProcessOutputs.groovy     |  2 +-
 2 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/CombineManyOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/CombineManyOp.groovy
index ec2a2a14ab..d709dbbc68 100644
--- a/modules/nextflow/src/main/groovy/nextflow/extension/CombineManyOp.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/extension/CombineManyOp.groovy
@@ -39,17 +39,20 @@ class CombineManyOp {
 
     private List iterators
 
-    private boolean singleton
-
     private List queues = []
 
+    private List singletons
+
+    private boolean emitSingleton
+
     private transient List combinations
 
     CombineManyOp(List sources, List iterators) {
         this.sources = sources
         this.iterators = iterators
-        this.singleton = iterators.size() == 0 && sources.every(ch -> !CH.isChannelQueue(ch))
-        this.queues = sources.collect( it -> [] )
+        this.queues = sources.collect( ch -> [] )
+        this.singletons = sources.collect( ch -> !CH.isChannelQueue(ch) )
+        this.emitSingleton = iterators.size() == 0 && singletons.every()
     }
 
     private Map handler(int index, DataflowWriteChannel target, AtomicInteger counter) {
@@ -58,7 +61,7 @@ class CombineManyOp {
             onNext(target, index, it)
         }
         opts.onComplete = {
-            if( counter.decrementAndGet() == 0 && !singleton )
+            if( counter.decrementAndGet() == 0 && !emitSingleton )
                 target.bind(Channel.STOP)
         }
         return opts
@@ -73,7 +76,9 @@ class CombineManyOp {
 
         // emit the next item if there are no iterators
         if( iterators.size() == 0 ) {
-            final args = queues.collect(q -> q.pop())
+            final args = (0..
+                singletons[i] ? queues[i].first() : queues[i].pop()
+            )
             target.bind(args)
             return
         }
@@ -82,7 +87,11 @@ class CombineManyOp {
         if( combinations == null )
             combinations = iterators.collect( i -> queues[i].first() ).combinations()
 
-        final args = (0.. i in iterators ? null : queues[i].pop() )
+        final args = (0..
+            i in iterators
+                ? null
+                : singletons[i] ? queues[i].first() : queues[i].pop()
+        )
         for( List entries : combinations ) {
             for( int k = 0; k < entries.size(); k++ )
                 args[iterators[k]] = entries[k]
@@ -92,7 +101,7 @@ class CombineManyOp {
     }
 
     DataflowWriteChannel apply() {
-        final target = CH.create(singleton)
+        final target = CH.create(emitSingleton)
         final counter = new AtomicInteger(sources.size())
         for( int i = 0; i < sources.size(); i++ )
             DataflowHelper.subscribeImpl( sources[i], handler(i, target, counter) )
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessOutputs.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessOutputs.groovy
index fa41ebcea6..6e6c23ec7f 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessOutputs.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessOutputs.groovy
@@ -48,7 +48,7 @@ class ProcessOutputs implements List, Cloneable {
     }
 
     void setDefault() {
-        final param = new ProcessOutput(new LazyVar('stdout'), [:])
+        final param = new ProcessOutput(this, new LazyVar('stdout'), [:])
         param.setChannel(new DataflowQueue())
         params.add(param)
     }

From 1e77a22a70f5c53882de12861f04bdf026c30b70 Mon Sep 17 00:00:00 2001
From: Ben Sherman 
Date: Sat, 9 Dec 2023 21:08:45 -0600
Subject: [PATCH 19/36] Fix task hash (resume still not working)

Signed-off-by: Ben Sherman 
---
 .../nextflow/processor/TaskProcessor.groovy   | 20 ++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy
index cc5de94f66..0e504b4286 100644
--- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy
@@ -1558,14 +1558,11 @@ class TaskProcessor {
             keys << task.getContainerFingerprint()
 
         // add task inputs
-        final inputs = config.getInputs()
-        final inputVars = inputs.getNames() - inputs.getFiles()*.getName()
-        for( String var : inputVars ) {
-            keys.add(var)
-            keys.add(task.context.get(var))
-        }
+        final inputVars = getTaskInputVars(task)
+        if( inputVars )
+            keys.add(inputVars.entrySet())
         if( task.env )
-            keys.add(task.env)
+            keys.add(task.env.entrySet())
         if( task.inputFiles )
             keys.add(task.inputFiles)
         if( task.stdin )
@@ -1671,6 +1668,15 @@ class TaskProcessor {
         log.info(buffer.toString())
     }
 
+    protected Map getTaskInputVars(TaskRun task) {
+        final result = [:]
+        final inputs = config.getInputs()
+        final inputVars = inputs.getNames() - inputs.getFiles()*.getName()
+        for( String var : inputVars )
+            result.put(var, task.context.get(var))
+        return result
+    }
+
     protected Map getTaskGlobalVars(TaskRun task) {
         final result = task.getGlobalVars(ownerScript.getBinding())
         final directives = getTaskExtensionDirectiveVars(task)

From c00ee3f907e8148a93c294fe374c24c301e1938c Mon Sep 17 00:00:00 2001
From: Ben Sherman 
Date: Wed, 13 Dec 2023 11:23:06 -0600
Subject: [PATCH 20/36] Update tests

Signed-off-by: Ben Sherman 
---
 .../test/groovy/nextflow/dag/DAGTest.groovy   | 22 ++++++++++++-------
 .../PathArityAwareTest.groovy}                | 12 +++++-----
 .../nextflow/processor/TaskConfigTest.groovy  |  2 +-
 .../processor/TaskProcessorTest.groovy        |  4 +++-
 .../nextflow/processor/TaskRunTest.groovy     | 18 +++++++--------
 .../script/dsl/ProcessBuilderTest.groovy      | 14 ++++++------
 .../script/params/ParamsInTest.groovy         |  7 +++---
 .../script/params/ParamsOutTest.groovy        |  9 ++++----
 .../groovy/test/TestParser.groovy             |  8 -------
 9 files changed, 49 insertions(+), 47 deletions(-)
 rename modules/nextflow/src/test/groovy/nextflow/{script/params/ArityParamTest.groovy => processor/PathArityAwareTest.groovy} (84%)

diff --git a/modules/nextflow/src/test/groovy/nextflow/dag/DAGTest.groovy b/modules/nextflow/src/test/groovy/nextflow/dag/DAGTest.groovy
index 77cd0fde44..badb8ccb03 100644
--- a/modules/nextflow/src/test/groovy/nextflow/dag/DAGTest.groovy
+++ b/modules/nextflow/src/test/groovy/nextflow/dag/DAGTest.groovy
@@ -22,10 +22,10 @@ import groovyx.gpars.dataflow.DataflowChannel
 import groovyx.gpars.dataflow.DataflowQueue
 import groovyx.gpars.dataflow.DataflowBroadcast
 import groovyx.gpars.dataflow.DataflowVariable
-import nextflow.script.params.InputsList;
-import nextflow.script.params.InParam;
-import nextflow.script.params.OutputsList;
-import nextflow.script.params.OutParam;
+import nextflow.script.ProcessInput
+import nextflow.script.ProcessInputs
+import nextflow.script.ProcessOutput
+import nextflow.script.ProcessOutputs
 import nextflow.Session
 /**
  *
@@ -293,12 +293,18 @@ class DAGTest extends Specification {
 
         def dag = new DAG()
 
-        def pInList = new InputsList()
-        def ip1 = Mock(InParam) { rawChannel >> chC }
+        def pInList = new ProcessInputs()
+        def ip1 = Mock(ProcessInput) {
+            getChannel() >> chC
+            getName() >> 'in1'
+        }
         pInList.add( ip1 )
 
-        def pOutList = new OutputsList()
-        def op1 = Mock(OutParam) { getOutChannel() >> chE }
+        def pOutList = new ProcessOutputs()
+        def op1 = Mock(ProcessOutput) {
+            getChannel() >> chE
+            getName() >> 'out1'
+        }
         pOutList.add( op1 )
 
         when:
diff --git a/modules/nextflow/src/test/groovy/nextflow/script/params/ArityParamTest.groovy b/modules/nextflow/src/test/groovy/nextflow/processor/PathArityAwareTest.groovy
similarity index 84%
rename from modules/nextflow/src/test/groovy/nextflow/script/params/ArityParamTest.groovy
rename to modules/nextflow/src/test/groovy/nextflow/processor/PathArityAwareTest.groovy
index bd4f50ee29..0dd03ef6f9 100644
--- a/modules/nextflow/src/test/groovy/nextflow/script/params/ArityParamTest.groovy
+++ b/modules/nextflow/src/test/groovy/nextflow/processor/PathArityAwareTest.groovy
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package nextflow.script.params
+package nextflow.processor
 
 import spock.lang.Specification
 import spock.lang.Unroll
@@ -22,17 +22,17 @@ import spock.lang.Unroll
  *
  * @author Ben Sherman 
  */
-class ArityParamTest extends Specification {
+class PathArityAwareTest extends Specification {
 
-    static class DefaultArityParam implements ArityParam {
-        DefaultArityParam() {}
+    static class PathArity implements PathArityAware {
+        PathArity() {}
     }
 
     @Unroll
     def testArity () {
 
         when:
-        def param = new DefaultArityParam()
+        def param = new PathArity()
         param.setArity(VALUE)
         then:
         param.arity.min == MIN
@@ -50,7 +50,7 @@ class ArityParamTest extends Specification {
     def testArityRange () {
 
         when:
-        def range = new ArityParam.Range(MIN, MAX)
+        def range = new PathArityAware.Range(MIN, MAX)
         then:
         range.contains(2) == TWO
         range.toString() == STRING
diff --git a/modules/nextflow/src/test/groovy/nextflow/processor/TaskConfigTest.groovy b/modules/nextflow/src/test/groovy/nextflow/processor/TaskConfigTest.groovy
index d76e6055a0..e96343285f 100644
--- a/modules/nextflow/src/test/groovy/nextflow/processor/TaskConfigTest.groovy
+++ b/modules/nextflow/src/test/groovy/nextflow/processor/TaskConfigTest.groovy
@@ -504,7 +504,7 @@ class TaskConfigTest extends Specification {
 
         when:
         config = new TaskConfig()
-        config.publishDir = [ [path: "${-> foo }/${-> bar }", mode: "${-> x }"] ] as ConfigList
+        config.publishDir = [ [path: "${-> foo }/${-> bar }", mode: "${-> x }"] ] as LazyList
         config.setContext( foo: 'world', bar: 'hello', x: 'copy' )
         then:
         config.getPublishDir() == [ PublishDir.create(path: 'world/hello', mode: 'copy') ]
diff --git a/modules/nextflow/src/test/groovy/nextflow/processor/TaskProcessorTest.groovy b/modules/nextflow/src/test/groovy/nextflow/processor/TaskProcessorTest.groovy
index 783cfa0848..d0ea88e771 100644
--- a/modules/nextflow/src/test/groovy/nextflow/processor/TaskProcessorTest.groovy
+++ b/modules/nextflow/src/test/groovy/nextflow/processor/TaskProcessorTest.groovy
@@ -23,6 +23,7 @@ import java.nio.file.Paths
 import java.util.concurrent.ExecutorService
 
 import groovyx.gpars.agent.Agent
+import groovyx.gpars.dataflow.DataflowReadChannel
 import nextflow.Global
 import nextflow.ISession
 import nextflow.Session
@@ -59,7 +60,8 @@ class TaskProcessorTest extends Specification {
             super(name, new NopeExecutor(session: session), session, script, taskConfig, new BodyDef({}, '..'))
         }
 
-        @Override protected void createOperator() { }
+        @Override
+        protected void createOperator(DataflowReadChannel source) { }
     }
 
 
diff --git a/modules/nextflow/src/test/groovy/nextflow/processor/TaskRunTest.groovy b/modules/nextflow/src/test/groovy/nextflow/processor/TaskRunTest.groovy
index e62f4c8263..d2d3412b54 100644
--- a/modules/nextflow/src/test/groovy/nextflow/processor/TaskRunTest.groovy
+++ b/modules/nextflow/src/test/groovy/nextflow/processor/TaskRunTest.groovy
@@ -29,7 +29,7 @@ import nextflow.file.FileHolder
 import nextflow.script.BodyDef
 import nextflow.script.ScriptBinding
 import nextflow.script.TaskClosure
-import nextflow.script.TokenVar
+import nextflow.util.LazyVar
 import nextflow.script.params.EnvInParam
 import nextflow.script.params.EnvOutParam
 import nextflow.script.params.FileInParam
@@ -63,8 +63,8 @@ class TaskRunTest extends Specification {
         def list = []
 
         task.setInput( new StdInParam(binding,list) )
-        task.setInput( new FileInParam(binding, list).bind(new TokenVar('x')), 'file1' )
-        task.setInput( new FileInParam(binding, list).bind(new TokenVar('y')), 'file2' )
+        task.setInput( new FileInParam(binding, list).bind(new LazyVar('x')), 'file1' )
+        task.setInput( new FileInParam(binding, list).bind(new LazyVar('y')), 'file2' )
         task.setInput( new EnvInParam(binding, list).bind('z'), 'env' )
 
 
@@ -119,7 +119,7 @@ class TaskRunTest extends Specification {
         def task = new TaskRun()
         def list = []
 
-        def x = new ValueInParam(binding, list).bind( new TokenVar('x') )
+        def x = new ValueInParam(binding, list).bind( new LazyVar('x') )
         def y = new FileInParam(binding, list).bind('y')
 
         task.setInput(x, 1)
@@ -137,7 +137,7 @@ class TaskRunTest extends Specification {
         def task = new TaskRun()
         def list = []
 
-        def x = new ValueInParam(binding, list).bind( new TokenVar('x') )
+        def x = new ValueInParam(binding, list).bind( new LazyVar('x') )
         def y = new FileInParam(binding, list).bind('y')
         def z = new FileInParam(binding, list).bind('z')
 
@@ -159,7 +159,7 @@ class TaskRunTest extends Specification {
         def list = []
 
         when:
-        def i1 = new ValueInParam(binding, list).bind( new TokenVar('x') )
+        def i1 = new ValueInParam(binding, list).bind( new LazyVar('x') )
         def o1 = new FileOutParam(binding,list).bind('file_out.alpha')
         def o2 = new ValueOutParam(binding,list).bind( 'x' )
         def o3 = new FileOutParam(binding,list).bind('file_out.beta')
@@ -203,7 +203,7 @@ class TaskRunTest extends Specification {
          * file with parametric name => true
          */
         when:
-        def s3 = new FileOutParam(binding, list).bind( new TokenVar('y') )
+        def s3 = new FileOutParam(binding, list).bind( new LazyVar('y') )
         def task3 = new TaskRun()
         task3.setOutput(s3)
         then:
@@ -687,8 +687,8 @@ class TaskRunTest extends Specification {
 
     def 'should return output env names' () {
         given:
-        def env1 = new EnvOutParam(new Binding(),[]).bind(new TokenVar('FOO'))
-        def env2 = new EnvOutParam(new Binding(),[]).bind(new TokenVar('BAR'))
+        def env1 = new EnvOutParam(new Binding(),[]).bind(new LazyVar('FOO'))
+        def env2 = new EnvOutParam(new Binding(),[]).bind(new LazyVar('BAR'))
         def task = new TaskRun()
         task.outputs.put(env1, null)
         task.outputs.put(env2, null)
diff --git a/modules/nextflow/src/test/groovy/nextflow/script/dsl/ProcessBuilderTest.groovy b/modules/nextflow/src/test/groovy/nextflow/script/dsl/ProcessBuilderTest.groovy
index 198b6150a1..b404796d4e 100644
--- a/modules/nextflow/src/test/groovy/nextflow/script/dsl/ProcessBuilderTest.groovy
+++ b/modules/nextflow/src/test/groovy/nextflow/script/dsl/ProcessBuilderTest.groovy
@@ -26,7 +26,7 @@ import nextflow.script.params.StdOutParam
 import nextflow.script.params.ValueInParam
 import nextflow.script.BaseScript
 import nextflow.script.ProcessConfig
-import nextflow.script.TokenVar
+import nextflow.util.LazyVar
 import nextflow.util.Duration
 import nextflow.util.MemoryUnit
 /**
@@ -114,9 +114,9 @@ class ProcessBuilderTest extends Specification {
 
         when:
         builder._out_stdout()
-        builder._out_file(new TokenVar('file1')).setInto('ch1')
-        builder._out_file(new TokenVar('file2')).setInto('ch2')
-        builder._out_file(new TokenVar('file3')).setInto('ch3')
+        builder._out_file(new LazyVar('file1')).setInto('ch1')
+        builder._out_file(new LazyVar('file2')).setInto('ch2')
+        builder._out_file(new LazyVar('file3')).setInto('ch3')
 
         then:
         config.outputs.size() == 4
@@ -583,21 +583,21 @@ class ProcessBuilderTest extends Specification {
 
         when:
         def config = new ProcessConfig(label: ['foo', 'other'])
-        new ProcessBuilder(config).applyConfig(settings, "processName", null, null)
+        new ProcessConfigBuilder(config).applyConfig(settings, "processName", null, null)
         then:
         config.cpus == 2
         config.disk == '100.GB'
 
         when:
         config = new ProcessConfig(label: ['foo', 'other', 'nodisk_label'])
-        new ProcessBuilder(config).applyConfig(settings, "processName", null, null)
+        new ProcessConfigBuilder(config).applyConfig(settings, "processName", null, null)
         then:
         config.cpus == 2
         !config.disk
 
         when:
         config = new ProcessConfig(label: ['other', 'nodisk_label'])
-        new ProcessBuilder(config).applyConfig(settings, "processName", null, null)
+        new ProcessConfigBuilder(config).applyConfig(settings, "processName", null, null)
         then:
         config.cpus == 4
         !config.disk
diff --git a/modules/nextflow/src/test/groovy/nextflow/script/params/ParamsInTest.groovy b/modules/nextflow/src/test/groovy/nextflow/script/params/ParamsInTest.groovy
index 7e5da2b208..fac12e95d2 100644
--- a/modules/nextflow/src/test/groovy/nextflow/script/params/ParamsInTest.groovy
+++ b/modules/nextflow/src/test/groovy/nextflow/script/params/ParamsInTest.groovy
@@ -24,6 +24,7 @@ import groovyx.gpars.dataflow.DataflowQueue
 import groovyx.gpars.dataflow.DataflowVariable
 import nextflow.Channel
 import nextflow.exception.ScriptRuntimeException
+import nextflow.script.PathArityAware
 import nextflow.processor.TaskProcessor
 import spock.lang.Timeout
 import test.Dsl2Spec
@@ -738,21 +739,21 @@ class ParamsInTest extends Dsl2Spec {
         in0.inChannel.val == FILE
         in0.index == 0
         in0.isPathQualifier()
-        in0.arity == new ArityParam.Range(1, 1)
+        in0.arity == new PathArityAware.Range(1, 1)
 
         in1.name == 'f1'
         in1.filePattern == '*'
         in1.inChannel.val == FILE
         in1.index == 1
         in1.isPathQualifier()
-        in1.arity == new ArityParam.Range(1, 2)
+        in1.arity == new PathArityAware.Range(1, 2)
 
         in2.name == '*.fa'
         in2.filePattern == '*.fa'
         in2.inChannel.val == FILE
         in2.index == 2
         in2.isPathQualifier()
-        in2.arity == new ArityParam.Range(1, Integer.MAX_VALUE)
+        in2.arity == new PathArityAware.Range(1, Integer.MAX_VALUE)
 
         in3.name == 'file.txt'
         in3.filePattern == 'file.txt'
diff --git a/modules/nextflow/src/test/groovy/nextflow/script/params/ParamsOutTest.groovy b/modules/nextflow/src/test/groovy/nextflow/script/params/ParamsOutTest.groovy
index d19820f4f6..a3225d3f05 100644
--- a/modules/nextflow/src/test/groovy/nextflow/script/params/ParamsOutTest.groovy
+++ b/modules/nextflow/src/test/groovy/nextflow/script/params/ParamsOutTest.groovy
@@ -21,8 +21,9 @@ import static test.TestParser.*
 import java.nio.file.Path
 
 import groovyx.gpars.dataflow.DataflowVariable
+import nextflow.script.PathArityAware
 import nextflow.processor.TaskContext
-import nextflow.script.TokenVar
+import nextflow.util.LazyVar
 import nextflow.util.BlankSeparatedList
 import test.Dsl2Spec
 /**
@@ -658,7 +659,7 @@ class ParamsOutTest extends Dsl2Spec {
          * val x
          */
         when:
-        param.target = new TokenVar('x')
+        param.target = new LazyVar('x')
         then:
         param.resolve(createTaskContext([x:'foo'])) == 'foo'
 
@@ -965,7 +966,7 @@ class ParamsOutTest extends Dsl2Spec {
         !out0.getGlob()
         !out0.getOptional()
         !out0.getIncludeInputs()
-        out0.getArity() == new ArityParam.Range(1, 1)
+        out0.getArity() == new PathArityAware.Range(1, 1)
 
         and:
         out1.getMaxDepth() == 5
@@ -976,7 +977,7 @@ class ParamsOutTest extends Dsl2Spec {
         out1.getGlob()
         out1.getOptional()
         out1.getIncludeInputs()
-        out1.getArity() == new ArityParam.Range(0, Integer.MAX_VALUE)
+        out1.getArity() == new PathArityAware.Range(0, Integer.MAX_VALUE)
     }
 
     def 'should set file options' () {
diff --git a/modules/nextflow/src/testFixtures/groovy/test/TestParser.groovy b/modules/nextflow/src/testFixtures/groovy/test/TestParser.groovy
index 60d46a9f01..edb3ac2aef 100644
--- a/modules/nextflow/src/testFixtures/groovy/test/TestParser.groovy
+++ b/modules/nextflow/src/testFixtures/groovy/test/TestParser.groovy
@@ -80,14 +80,6 @@ class TestParser {
 
     @InheritConstructors
     static class TestTaskProcessor extends TaskProcessor {
-        @Override
-        def run () {
-            // this is needed to mimic the out channels normalisation
-            // made by the real 'run' method - check the superclass
-            if ( config.getOutputs().size() == 0 ) {
-                config.fakeOutput()
-            }
-        }
     }
 
     @InheritConstructors

From f7b3fa86cf4bd4061a06ea00ea11fae3ae4614c4 Mon Sep 17 00:00:00 2001
From: Ben Sherman 
Date: Wed, 13 Dec 2023 14:24:36 -0600
Subject: [PATCH 21/36] Move annotation API to separate branch

Signed-off-by: Ben Sherman 
---
 .../src/main/groovy/nextflow/Nextflow.groovy  |   10 +-
 .../nextflow/ast/BinaryExpressionXform.groovy |  119 --
 .../groovy/nextflow/ast/DslCodeVistor.groovy  | 1248 ----------------
 .../{ProcessFn.groovy => NextflowDSL.groovy}  |   23 +-
 .../nextflow/ast/NextflowDSLImpl.groovy       | 1272 +++++++++++++++++
 .../nextflow/ast/NextflowXformImpl.groovy     |   94 +-
 .../ast/{WorkflowFn.groovy => OpXform.groovy} |   16 +-
 ...peratorXform.groovy => OpXformImpl.groovy} |  137 +-
 .../groovy/nextflow/ast/ProcessFnXform.groovy |  167 ---
 .../nextflow/ast/WorkflowFnXform.groovy       |   78 -
 .../nextflow/config/ConfigParser.groovy       |    2 +
 .../config/ConfigTransformImpl.groovy         |    2 -
 .../nextflow/processor/TaskConfig.groovy      |    4 +-
 .../nextflow/processor/TaskProcessor.groovy   |    9 +-
 .../groovy/nextflow/processor/TaskRun.groovy  |   21 +-
 .../groovy/nextflow/script/BaseScript.groovy  |  107 +-
 .../groovy/nextflow/script/ChannelOut.groovy  |    6 +-
 .../nextflow/script/ProcessConfig.groovy      |    6 +-
 .../groovy/nextflow/script/ProcessDef.groovy  |   58 +-
 .../groovy/nextflow/script/ScriptMeta.groovy  |    4 -
 .../nextflow/script/ScriptParser.groovy       |    8 +-
 .../nextflow/script/ScriptTokens.groovy       |    4 +-
 .../groovy/nextflow/script/WorkflowDef.groovy |   48 +-
 .../nextflow/script/dsl/ProcessBuilder.groovy |    6 +-
 .../script/dsl/ProcessInputsBuilder.groovy    |   71 -
 .../script/dsl/ProcessOutputsBuilder.groovy   |   76 -
 ...Test.groovy => NextflowDSLImplTest.groovy} |    2 +-
 ...torXformTest.groovy => OpXformTest.groovy} |    4 +-
 .../nextflow/script/IncludeDefTest.groovy     |    4 +-
 .../nextflow/script/WorkflowDefTest.groovy    |   16 +-
 .../script/params/ParamsDsl2Test.groovy       |    6 +-
 31 files changed, 1528 insertions(+), 2100 deletions(-)
 delete mode 100644 modules/nextflow/src/main/groovy/nextflow/ast/BinaryExpressionXform.groovy
 delete mode 100644 modules/nextflow/src/main/groovy/nextflow/ast/DslCodeVistor.groovy
 rename modules/nextflow/src/main/groovy/nextflow/ast/{ProcessFn.groovy => NextflowDSL.groovy} (66%)
 create mode 100644 modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSLImpl.groovy
 rename modules/nextflow/src/main/groovy/nextflow/ast/{WorkflowFn.groovy => OpXform.groovy} (77%)
 rename modules/nextflow/src/main/groovy/nextflow/ast/{OperatorXform.groovy => OpXformImpl.groovy} (90%)
 delete mode 100644 modules/nextflow/src/main/groovy/nextflow/ast/ProcessFnXform.groovy
 delete mode 100644 modules/nextflow/src/main/groovy/nextflow/ast/WorkflowFnXform.groovy
 delete mode 100644 modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessInputsBuilder.groovy
 delete mode 100644 modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessOutputsBuilder.groovy
 rename modules/nextflow/src/test/groovy/nextflow/ast/{DslCodeVisitorTest.groovy => NextflowDSLImplTest.groovy} (98%)
 rename modules/nextflow/src/test/groovy/nextflow/ast/{OperatorXformTest.groovy => OpXformTest.groovy} (98%)

diff --git a/modules/nextflow/src/main/groovy/nextflow/Nextflow.groovy b/modules/nextflow/src/main/groovy/nextflow/Nextflow.groovy
index 97d5988b32..5976f229e3 100644
--- a/modules/nextflow/src/main/groovy/nextflow/Nextflow.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/Nextflow.groovy
@@ -24,6 +24,8 @@ import java.nio.file.NoSuchFileException
 import java.nio.file.Path
 
 import groovyx.gpars.dataflow.DataflowReadChannel
+import nextflow.ast.OpXform
+import nextflow.ast.OpXformImpl
 import nextflow.exception.StopSplitIterationException
 import nextflow.exception.WorkflowScriptErrorException
 import nextflow.extension.GroupKey
@@ -379,11 +381,11 @@ class Nextflow {
      * Marker method to create a closure to be passed to {@link OperatorImpl#branch(DataflowReadChannel, groovy.lang.Closure)}
      * operator.
      *
-     * Despite apparently is doing nothing, this method is needed as marker to apply the {@link OperatorXform} AST
+     * Despite apparently is doing nothing, this method is needed as marker to apply the {@link OpXform} AST
      * transformation required to interpret the closure content as required for the branch evaluation.
      *
      * @see OperatorImpl#branch(DataflowReadChannel, Closure)
-     * @see OperatorXform
+     * @see OpXformImpl
      *
      * @param closure
      * @return
@@ -394,11 +396,11 @@ class Nextflow {
      * Marker method to create a closure to be passed to {@link OperatorImpl#fork(DataflowReadChannel, Closure)}
      * operator.
      *
-     * Despite apparently is doing nothing, this method is needed as marker to apply the {@link OperatorXform} AST
+     * Despite apparently is doing nothing, this method is needed as marker to apply the {@link OpXform} AST
      * transformation required to interpret the closure content as required for the branch evaluation.
      *
      * @see OperatorImpl#multiMap(groovyx.gpars.dataflow.DataflowReadChannel, groovy.lang.Closure) (DataflowReadChannel, Closure)
-     * @see OperatorXform
+     * @see OpXformImpl
      *
      * @param closure
      * @return
diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/BinaryExpressionXform.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/BinaryExpressionXform.groovy
deleted file mode 100644
index ee5c282759..0000000000
--- a/modules/nextflow/src/main/groovy/nextflow/ast/BinaryExpressionXform.groovy
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * Copyright 2013-2023, Seqera Labs
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package nextflow.ast
-
-import groovy.transform.CompileStatic
-import groovy.util.logging.Slf4j
-import org.codehaus.groovy.ast.ClassCodeExpressionTransformer
-import org.codehaus.groovy.ast.expr.BinaryExpression
-import org.codehaus.groovy.ast.expr.ClosureExpression
-import org.codehaus.groovy.ast.expr.Expression
-import org.codehaus.groovy.ast.expr.MethodCallExpression
-import org.codehaus.groovy.ast.expr.NotExpression
-import org.codehaus.groovy.ast.tools.GeneralUtils
-import org.codehaus.groovy.control.SourceUnit
-/**
- * Implements Nextflow Xform logic
-
- * See http://groovy-lang.org/metaprogramming.html#_classcodeexpressiontransformer
- *
- * @author Paolo Di Tommaso 
- */
-@Slf4j
-@CompileStatic
-class BinaryExpressionXform extends ClassCodeExpressionTransformer {
-
-    private final SourceUnit unit
-
-    BinaryExpressionXform(SourceUnit unit) {
-        this.unit = unit
-    }
-
-    @Override
-    protected SourceUnit getSourceUnit() { unit }
-
-    @Override
-    Expression transform(Expression expr) {
-        if (expr == null)
-            return null
-
-        def newExpr = transformBinaryExpression(expr)
-        if( newExpr ) {
-            return newExpr
-        }
-        else if( expr instanceof ClosureExpression) {
-            visitClosureExpression(expr)
-        }
-
-        return super.transform(expr)
-    }
-
-    /**
-     * This method replaces the `==` with the invocation of
-     * {@link LangHelpers#compareEqual(java.lang.Object, java.lang.Object)}
-     *
-     * This is required to allow the comparisons of `Path` objects
-     * which by default are not supported because it implements the Comparator interface
-     *
-     * See
-     *  {@link LangHelpers#compareEqual(java.lang.Object, java.lang.Object)}
-     *  https://stackoverflow.com/questions/28355773/in-groovy-why-does-the-behaviour-of-change-for-interfaces-extending-compar#comment45123447_28387391
-     *
-     */
-    protected Expression transformBinaryExpression(Expression expr) {
-
-        if( expr.class != BinaryExpression )
-            return null
-
-        def binary = expr as BinaryExpression
-        def left = binary.getLeftExpression()
-        def right = binary.getRightExpression()
-
-        if( '=='.equals(binary.operation.text) )
-            return call('compareEqual',left,right)
-
-        if( '!='.equals(binary.operation.text) )
-            return new NotExpression(call('compareEqual',left,right))
-
-        if( '<'.equals(binary.operation.text) )
-            return call('compareLessThan', left,right)
-
-        if( '<='.equals(binary.operation.text) )
-            return call('compareLessThanEqual', left,right)
-
-        if( '>'.equals(binary.operation.text) )
-            return call('compareGreaterThan', left,right)
-
-        if( '>='.equals(binary.operation.text) )
-            return call('compareGreaterThanEqual', left,right)
-
-        return null
-    }
-
-
-    private MethodCallExpression call(String method, Expression left, Expression right) {
-
-        final a = transformBinaryExpression(left) ?: left
-        final b = transformBinaryExpression(right) ?: right
-
-        GeneralUtils.callX(
-                GeneralUtils.classX(LangHelpers),
-                method,
-                GeneralUtils.args(a,b))
-    }
-
-}
diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/DslCodeVistor.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/DslCodeVistor.groovy
deleted file mode 100644
index d3d667c443..0000000000
--- a/modules/nextflow/src/main/groovy/nextflow/ast/DslCodeVistor.groovy
+++ /dev/null
@@ -1,1248 +0,0 @@
-/*
- * Copyright 2013-2023, Seqera Labs
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package nextflow.ast
-
-import static nextflow.Const.*
-import static nextflow.ast.ASTHelpers.*
-import static org.codehaus.groovy.ast.tools.GeneralUtils.*
-
-import groovy.transform.CompileStatic
-import groovy.util.logging.Slf4j
-import nextflow.NF
-import nextflow.script.BaseScript
-import nextflow.script.BodyDef
-import nextflow.script.IncludeDef
-import nextflow.script.TaskClosure
-import nextflow.script.TokenEnvCall
-import nextflow.script.TokenFileCall
-import nextflow.script.TokenPathCall
-import nextflow.script.TokenStdinCall
-import nextflow.script.TokenStdoutCall
-import nextflow.script.TokenValCall
-import nextflow.script.TokenValRef
-import nextflow.util.LazyVar
-import org.codehaus.groovy.ast.ASTNode
-import org.codehaus.groovy.ast.ClassCodeVisitorSupport
-import org.codehaus.groovy.ast.MethodNode
-import org.codehaus.groovy.ast.Parameter
-import org.codehaus.groovy.ast.VariableScope
-import org.codehaus.groovy.ast.expr.ArgumentListExpression
-import org.codehaus.groovy.ast.expr.BinaryExpression
-import org.codehaus.groovy.ast.expr.CastExpression
-import org.codehaus.groovy.ast.expr.ClosureExpression
-import org.codehaus.groovy.ast.expr.ConstantExpression
-import org.codehaus.groovy.ast.expr.Expression
-import org.codehaus.groovy.ast.expr.GStringExpression
-import org.codehaus.groovy.ast.expr.ListExpression
-import org.codehaus.groovy.ast.expr.MapEntryExpression
-import org.codehaus.groovy.ast.expr.MapExpression
-import org.codehaus.groovy.ast.expr.MethodCallExpression
-import org.codehaus.groovy.ast.expr.PropertyExpression
-import org.codehaus.groovy.ast.expr.TupleExpression
-import org.codehaus.groovy.ast.expr.UnaryMinusExpression
-import org.codehaus.groovy.ast.expr.VariableExpression
-import org.codehaus.groovy.ast.stmt.BlockStatement
-import org.codehaus.groovy.ast.stmt.ExpressionStatement
-import org.codehaus.groovy.ast.stmt.ReturnStatement
-import org.codehaus.groovy.ast.stmt.Statement
-import org.codehaus.groovy.control.SourceUnit
-import org.codehaus.groovy.syntax.SyntaxException
-import org.codehaus.groovy.syntax.Token
-import org.codehaus.groovy.syntax.Types
-/**
- * Implements the syntax transformations for Nextflow DSL2.
- *
- * @author Paolo Di Tommaso 
- */
-@Slf4j
-@CompileStatic
-class DslCodeVisitor extends ClassCodeVisitorSupport {
-
-    final static private String WORKFLOW_TAKE = 'take'
-    final static private String WORKFLOW_EMIT = 'emit'
-    final static private String WORKFLOW_MAIN = 'main'
-    final static private List SCOPES = [WORKFLOW_TAKE, WORKFLOW_EMIT, WORKFLOW_MAIN]
-
-    final static public String PROCESS_WHEN = 'when'
-    final static public String PROCESS_STUB = 'stub'
-
-    static public String OUT_PREFIX = '$out'
-
-    static private Set RESERVED_NAMES
-
-    static {
-        // method names implicitly defined by the groovy script SHELL
-        RESERVED_NAMES = ['main','run','runScript'] as Set
-        // existing method cannot be used for custom script definition
-        for( def method : BaseScript.getMethods() ) {
-            RESERVED_NAMES.add(method.name)
-        }
-
-    }
-
-    final private SourceUnit unit
-
-    private String currentTaskName
-
-    private String currentLabel
-
-    private String bodyLabel
-
-    private Set processNames = []
-
-    private Set workflowNames = []
-
-    private Set functionNames = []
-
-    private int anonymousWorkflow
-
-    @Override
-    protected SourceUnit getSourceUnit() { unit }
-
-    DslCodeVisitor(SourceUnit unit) {
-        this.unit = unit
-    }
-
-    @Override
-    void visitMethod(MethodNode node) {
-        if( node.public && !node.static && !node.synthetic && !node.metaDataMap?.'org.codehaus.groovy.ast.MethodNode.isScriptBody') {
-            if( !isIllegalName(node.name, node))
-                functionNames.add(node.name)
-        }
-        super.visitMethod(node)
-    }
-
-    @Override
-    void visitMethodCallExpression(MethodCallExpression methodCall) {
-        // pre-condition to be verified to apply the transformation
-        final preCondition = methodCall.objectExpression?.getText() == 'this'
-        final methodName = methodCall.getMethodAsString()
-
-        /*
-         * intercept the *process* method in order to transform the script closure
-         */
-        if( methodName == 'process' && preCondition ) {
-
-            // clear block label
-            bodyLabel = null
-            currentLabel = null
-            currentTaskName = methodName
-            try {
-                convertProcessDef(methodCall,sourceUnit)
-                super.visitMethodCallExpression(methodCall)
-            }
-            finally {
-                currentTaskName = null
-            }
-        }
-        else if( methodName == 'workflow' && preCondition ) {
-            convertWorkflowDef(methodCall,sourceUnit)
-            super.visitMethodCallExpression(methodCall)
-        }
-
-        // just apply the default behavior
-        else {
-            super.visitMethodCallExpression(methodCall)
-        }
-
-    }
-
-    @Override
-    void visitExpressionStatement(ExpressionStatement stm) {
-        if( stm.text.startsWith('this.include(') && stm.getExpression() instanceof MethodCallExpression )  {
-            final methodCall = (MethodCallExpression)stm.getExpression()
-            convertIncludeDef(methodCall)
-            // this is necessary to invoke the `load` method on the include definition
-            final loadCall = new MethodCallExpression(methodCall, 'load0', new ArgumentListExpression(new VariableExpression('params')))
-            stm.setExpression(loadCall)
-        }
-        super.visitExpressionStatement(stm)
-    }
-
-    protected void convertIncludeDef(MethodCallExpression call) {
-        if( call.methodAsString=='include' && call.arguments instanceof ArgumentListExpression ) {
-            final allArgs = (ArgumentListExpression)call.arguments
-            if( allArgs.size() != 1 ) {
-                syntaxError(call, "Not a valid include definition -- it must specify the module path")
-                return
-            }
-
-            final arg = allArgs[0]
-            final newArgs = new ArgumentListExpression()
-            if( arg instanceof ConstantExpression ) {
-                newArgs.addExpression( createX(IncludeDef, arg) )
-            }
-            else if( arg instanceof VariableExpression ) {
-                // the name of the component i.e. process, workflow, etc to import
-                final component = arg.getName()
-                // wrap the name in a `LazyVar` type
-                final token = createX(LazyVar, new ConstantExpression(component))
-                // create a new `IncludeDef` object
-                newArgs.addExpression(createX(IncludeDef, token))
-            }
-            else if( arg instanceof CastExpression && arg.getExpression() instanceof VariableExpression) {
-                def cast = (CastExpression)arg
-                // the name of the component i.e. process, workflow, etc to import
-                final component = (cast.expression as VariableExpression).getName()
-                // wrap the name in a `LazyVar` type
-                final token = createX(LazyVar, new ConstantExpression(component))
-                // the alias to give it
-                final alias = constX(cast.type.name)
-                newArgs.addExpression( createX(IncludeDef, token, alias) )
-            }
-            else if( arg instanceof ClosureExpression ) {
-                // multiple modules inclusion 
-                final block = (BlockStatement)arg.getCode()
-                final modulesList = new ListExpression()
-                for( Statement stm : block.statements ) {
-                    if( stm instanceof ExpressionStatement ) {
-                        CastExpression castX
-                        VariableExpression varX
-                        Expression moduleX
-                        if( (varX=isVariableX(stm.expression)) ) {
-                            def name = constX(varX.name)
-                            moduleX = createX(IncludeDef.Module, name)
-                        }
-                        else if( (castX=isCastX(stm.expression)) && (varX=isVariableX(castX.expression)) ) {
-                            def name = constX(varX.name)
-                            final alias = constX(castX.type.name)
-                            moduleX = createX(IncludeDef.Module, name, alias)
-                        }
-                        else {
-                            syntaxError(call, "Not a valid include module name")
-                            return
-                        }
-                        modulesList.addExpression(moduleX)
-                    }
-                    else {
-                        syntaxError(call, "Not a valid include module name")
-                        return
-                    }
-
-                }
-                newArgs.addExpression( createX(IncludeDef, modulesList) )
-            }
-            else {
-                syntaxError(call, "Not a valid include definition -- it must specify the module path as a string")
-                return
-            }
-            call.setArguments(newArgs)
-        }
-        else if( call.objectExpression instanceof MethodCallExpression ) {
-            convertIncludeDef((MethodCallExpression)call.objectExpression)
-        }
-    }
-
-    /*
-     * this method transforms the DSL definition
-     *
-     *   workflow foo {
-     *     code
-     *   }
-     *
-     * into a method invocation as
-     *
-     *   workflow('foo', { -> code })
-     *
-     */
-    protected void convertWorkflowDef(MethodCallExpression methodCall, SourceUnit unit) {
-        log.trace "Convert 'workflow' ${methodCall.arguments}"
-
-        assert methodCall.arguments instanceof ArgumentListExpression
-        def args = (ArgumentListExpression)methodCall.arguments
-        def len = args.size()
-
-        // anonymous workflow definition
-        if( len == 1 && args[0] instanceof ClosureExpression ) {
-            if( anonymousWorkflow++ > 0 ) {
-                unit.addError( new SyntaxException("Duplicate entry workflow definition", methodCall.lineNumber, methodCall.columnNumber+8))
-                return
-            }
-
-            def newArgs = new ArgumentListExpression()
-            def body = (ClosureExpression)args[0]
-            newArgs.addExpression( makeWorkflowDefWrapper(body,true) )
-            methodCall.setArguments( newArgs )
-            return 
-        }
-
-        // extract the first argument which has to be a method-call expression
-        // the name of this method represent the *workflow* name
-        if( len != 1 || !args[0].class.isAssignableFrom(MethodCallExpression) ) {
-            log.debug "Missing name in workflow definition at line: ${methodCall.lineNumber}"
-            unit.addError( new SyntaxException("Workflow definition syntax error -- A string identifier must be provided after the `workflow` keyword", methodCall.lineNumber, methodCall.columnNumber+8))
-            return
-        }
-
-        final nested = args[0] as MethodCallExpression
-        final name = nested.getMethodAsString()
-        // check the process name is not defined yet
-        if( isIllegalName(name, methodCall) ) {
-            return
-        }
-        workflowNames.add(name)
-
-        // the nested method arguments are the arguments to be passed
-        // to the process definition, plus adding the process *name*
-        // as an extra item in the arguments list
-        args = (ArgumentListExpression)nested.getArguments()
-        len = args.size()
-        log.trace "Workflow name: $name with args: $args"
-
-        // make sure to add the 'name' after the map item
-        // (which represent the named parameter attributes)
-        def newArgs = new ArgumentListExpression()
-
-        // add the workflow body def
-        if( len != 1 || !(args[0] instanceof ClosureExpression)) {
-            syntaxError(methodCall, "Invalid workflow definition")
-            return
-        }
-
-        final body = (ClosureExpression)args[0]
-        newArgs.addExpression( constX(name) )
-        newArgs.addExpression( makeWorkflowDefWrapper(body,false) )
-
-        // set the new list as the new arguments
-        methodCall.setArguments( newArgs )
-    }
-
-
-    protected Statement normWorkflowParam(ExpressionStatement stat, String type, Set uniqueNames, List body) {
-        MethodCallExpression callx
-        VariableExpression varx
-
-        if( (callx=isMethodCallX(stat.expression)) && isThisX(callx.objectExpression) ) {
-            final name = "_${type}_${callx.methodAsString}"
-            return stmt( callThisX(name, callx.arguments) )
-        }
-
-        if( (varx=isVariableX(stat.expression)) ) {
-            final name = "_${type}_${varx.name}"
-            return stmt( callThisX(name) )
-        }
-
-        if( type == WORKFLOW_EMIT ) {
-            return createAssignX(stat, body, type, uniqueNames)
-        }
-
-        syntaxError(stat, "Workflow malformed parameter definition")
-        return stat
-    }
-
-    protected Statement createAssignX(ExpressionStatement stat, List body, String type, Set uniqueNames) {
-        BinaryExpression binx
-        MethodCallExpression callx
-        Expression args=null
-
-        if( (binx=isAssignX(stat.expression)) ) {
-            // keep the statement in body to allow it to be evaluated
-            body.add(stat)
-            // and create method call expr to capture the var name in the emission
-            final left = (VariableExpression)binx.leftExpression
-            final name = "_${type}_${left.name}"
-            return stmt( callThisX(name) )
-        }
-
-        if( (callx=isMethodCallX(stat.expression)) && callx.objectExpression.text!='this' && hasTo(callx)) {
-            // keep the args
-            args = callx.arguments
-            // replace the method call expression with a property
-            stat.expression = new PropertyExpression(callx.objectExpression, callx.method)
-            // then, fallback to default case
-        }
-
-        // wrap the expression into a assignment expression
-        final var = getNextName(uniqueNames)
-        final left = new VariableExpression(var)
-        final right = stat.expression
-        final token = new Token(Types.ASSIGN, '=', -1, -1)
-        final assign = new BinaryExpression(left, token, right)
-        body.add(stmt(assign))
-
-        // the call method statement for the emit declaration
-        final name="_${type}_${var}"
-        callx =  args ? callThisX(name, args) : callThisX(name)
-        return stmt(callx)
-    }
-
-    protected boolean hasTo(MethodCallExpression callX) {
-        def tupleX = isTupleX(callX.arguments)
-        if( !tupleX ) return false
-        if( !tupleX.expressions ) return false
-        def mapX = isMapX(tupleX.expressions[0])
-        if( !mapX ) return false
-        def entry = mapX.getMapEntryExpressions().find { isConstX(it.keyExpression).text=='to' }
-        return entry != null
-    }
-
-    protected String getNextName(Set allNames) {
-        String result
-        while( true ) {
-            result = OUT_PREFIX + allNames.size()
-            if( allNames.add(result) )
-                break
-        }
-        return result
-    }
-
-    protected Expression makeWorkflowDefWrapper( ClosureExpression closure, boolean anonymous ) {
-
-        final codeBlock = (BlockStatement) closure.code
-        final codeStms = codeBlock.statements
-        final scope = codeBlock.variableScope
-
-        final visited = new HashMap(5);
-        final emitNames = new LinkedHashSet(codeStms.size())
-        final wrap = new ArrayList(codeStms.size())
-        final body = new ArrayList(codeStms.size())
-        final source = new StringBuilder()
-        String context = null
-        String previous = null
-        for( Statement stm : codeStms ) {
-            previous = context
-            context = stm.statementLabel ?: context
-            // check for changing context
-            if( context && context != previous ) {
-                if( visited[context] && visited[previous] ) {
-                    syntaxError(stm, "Unexpected workflow `${context}` context here")
-                    break
-                }
-            }
-            visited[context] = true
-
-            switch (context) {
-                case WORKFLOW_TAKE:
-                case WORKFLOW_EMIT:
-                    if( !(stm instanceof ExpressionStatement) ) {
-                        syntaxError(stm, "Workflow malformed parameter definition")
-                        break
-                    }
-                    wrap.add(normWorkflowParam(stm as ExpressionStatement, context, emitNames, body))
-                break
-
-                case WORKFLOW_MAIN:
-                    body.add(stm)
-                    break
-
-                default:
-                    if( context ) {
-                        def opts = SCOPES.closest(context)
-                        def msg = "Unknown execution scope '$context:'"
-                        if( opts ) msg += " -- Did you mean ${opts.collect{"'$it'"}.join(', ')}"
-                        syntaxError(stm, msg)
-                    }
-                    body.add(stm)
-            }
-        }
-        // read the closure source
-        readSource(closure, source, unit, true)
-
-        final bodyClosure = closureX(null, block(scope, body))
-        final invokeBody = makeScriptWrapper(bodyClosure, source.toString(), 'workflow', unit)
-        wrap.add( stmt(invokeBody) )
-
-        closureX(null, block(scope, wrap))
-    }
-
-    protected void syntaxError(ASTNode node, String message) {
-        int line = node.lineNumber
-        int coln = node.columnNumber
-        unit.addError( new SyntaxException(message,line,coln))
-    }
-
-    /**
-     * Transform a DSL `process` definition into a proper method invocation
-     *
-     * @param methodCall
-     * @param unit
-     */
-    protected void convertProcessBlock( MethodCallExpression methodCall, SourceUnit unit ) {
-        log.trace "Apply task closure transformation to method call: $methodCall"
-
-        final args = methodCall.arguments as ArgumentListExpression
-        final lastArg = args.expressions.size()>0 ? args.getExpression(args.expressions.size()-1) : null
-        final isClosure = lastArg instanceof ClosureExpression
-
-        if( isClosure ) {
-            // the block holding all the statements defined in the process (closure) definition
-            final block = (lastArg as ClosureExpression).code as BlockStatement
-
-            /*
-             * iterate over the list of statements to:
-             * - converts the method after the 'input:' label as input parameters
-             * - converts the method after the 'output:' label as output parameters
-             * - collect all the statement after the 'exec:' label
-             */
-            def source = new StringBuilder()
-            List execStatements = []
-
-            List whenStatements = []
-            def whenSource = new StringBuilder()
-
-            List stubStatements = []
-            def stubSource = new StringBuilder()
-
-
-            def iterator = block.getStatements().iterator()
-            while( iterator.hasNext() ) {
-
-                // get next statement
-                Statement stm = iterator.next()
-
-                // keep track of current block label
-                currentLabel = stm.statementLabel ?: currentLabel
-
-                switch(currentLabel) {
-                    case 'input':
-                        if( stm instanceof ExpressionStatement ) {
-                            fixLazyGString( stm )
-                            fixStdinStdout( stm )
-                            convertInputMethod( stm.getExpression() )
-                        }
-                        break
-
-                    case 'output':
-                        if( stm instanceof ExpressionStatement ) {
-                            fixLazyGString( stm )
-                            fixStdinStdout( stm )
-                            convertOutputMethod( stm.getExpression() )
-                        }
-                        break
-
-                    case 'exec':
-                        bodyLabel = currentLabel
-                        iterator.remove()
-                        execStatements << stm
-                        readSource(stm,source,unit)
-                        break
-
-                    case 'script':
-                    case 'shell':
-                        bodyLabel = currentLabel
-                        iterator.remove()
-                        execStatements << stm
-                        readSource(stm,source,unit)
-                        break
-
-                    case PROCESS_STUB:
-                        iterator.remove()
-                        stubStatements << stm
-                        readSource(stm,stubSource,unit)
-                        break
-
-                    // capture the statements in a when guard and remove from the current block
-                    case PROCESS_WHEN:
-                        if( iterator.hasNext() ) {
-                            iterator.remove()
-                            whenStatements << stm
-                            readSource(stm,whenSource,unit)
-                            break
-                        }
-                        // when entering in this branch means that this is the last statement,
-                        // which is supposed to be the task command
-                        // hence if no previous `when` statement has been processed, a syntax error is returned
-                        else if( !whenStatements ) {
-                            int line = methodCall.lineNumber
-                            int coln = methodCall.columnNumber
-                            unit.addError(new SyntaxException("Invalid process definition -- Empty `when` or missing `script` statement", line, coln))
-                            return
-                        }
-                        else
-                            break
-
-                    default:
-                        if(currentLabel) {
-                            def line = stm.getLineNumber()
-                            def coln = stm.getColumnNumber()
-                            unit.addError(new SyntaxException("Invalid process definition -- Unknown keyword `$currentLabel`",line,coln))
-                            return
-                        }
-
-                        fixLazyGString(stm)
-                        fixDirectiveWithNegativeValue(stm)  // Fixes #180
-                }
-            }
-
-            /*
-             * add the `when` block if found
-             */
-            if( whenStatements ) {
-                addWhenGuardCall(whenStatements, whenSource, block)
-            }
-
-            /*
-             * add try `stub` block if found
-             */
-            if( stubStatements ) {
-                final newBLock = addStubCall(stubStatements, stubSource, block)
-                newBLock.visit(new TaskCmdXformVisitor(unit))
-            }
-
-            /*
-             * wrap all the statements after the 'exec:'  label by a new closure containing them (in a new block)
-             */
-            final len = block.statements.size()
-            boolean done = false
-            if( execStatements ) {
-                // create a new Closure
-                def execBlock = new BlockStatement(execStatements, new VariableScope(block.variableScope))
-                def execClosure = new ClosureExpression( Parameter.EMPTY_ARRAY, execBlock )
-
-                // append the new block to the
-                // set the 'script' flag parameter
-                def wrap = makeScriptWrapper(execClosure, source, bodyLabel, unit)
-                block.addStatement( new ExpressionStatement(wrap) )
-                if( bodyLabel == 'script' )
-                    block.visit(new TaskCmdXformVisitor(unit))
-                done = true
-
-            }
-            // when only the `stub` block is defined add an empty command
-            else if ( !bodyLabel && stubStatements ) {
-                final cmd = 'true'
-                final list = new ArrayList(1);
-                list.add( new ExpressionStatement(constX(cmd)) )
-                final dummyBlock = new BlockStatement( list, new VariableScope(block.variableScope))
-                final dummyClosure = new ClosureExpression( Parameter.EMPTY_ARRAY, dummyBlock )
-
-                // append the new block to the
-                // set the 'script' flag parameter
-                final wrap = makeScriptWrapper(dummyClosure, cmd, 'script', unit)
-                block.addStatement( new ExpressionStatement(wrap) )
-                done = true
-            }
-
-            /*
-             * when the last statement is a string script, the 'script:' label can be omitted
-             */
-            else if( len ) {
-                def stm = block.getStatements().get(len-1)
-                readSource(stm,source,unit)
-
-                if ( stm instanceof ReturnStatement  ){
-                    done = wrapExpressionWithClosure(block, stm.getExpression(), len, source, unit)
-                }
-
-                else if ( stm instanceof ExpressionStatement )  {
-                    done = wrapExpressionWithClosure(block, stm.getExpression(), len, source, unit)
-                }
-
-                // apply command variables escape
-                stm.visit(new TaskCmdXformVisitor(unit))
-            }
-
-            if (!done) {
-                log.trace "Invalid 'process' definition -- Process must terminate with string expression"
-                int line = methodCall.lineNumber
-                int coln = methodCall.columnNumber
-                unit.addError( new SyntaxException("Invalid process definition -- Make sure the process ends with a script wrapped by quote characters",line,coln))
-            }
-        }
-    }
-
-    /**
-     * Converts a `when` block into a when method call expression. The when code is converted into a
-     * closure expression and set a `when` directive in the process configuration properties.
-     *
-     * See {@link nextflow.script.ProcessConfig#configProperties}
-     * See {@link nextflow.processor.TaskConfig#getGuard(java.lang.String)}
-     */
-    protected BlockStatement addWhenGuardCall( List statements, StringBuilder source, BlockStatement parent ) {
-        createBlock0(PROCESS_WHEN, statements, source, parent)
-    }
-
-    protected BlockStatement addStubCall(List statements, StringBuilder source, BlockStatement parent ) {
-        createBlock0(PROCESS_STUB, statements, source, parent)
-    }
-
-    protected BlockStatement createBlock0( String blockName, List statements, StringBuilder source, BlockStatement parent ) {
-        // wrap the code block into a closure expression
-        def block = new BlockStatement(statements, new VariableScope(parent.variableScope))
-        def closure = new ClosureExpression( Parameter.EMPTY_ARRAY, block )
-
-        // the closure expression is wrapped itself into a TaskClosure object
-        // in order to capture the closure source other than the closure code
-        List newArgs = []
-        newArgs << closure
-        newArgs << new ConstantExpression(source.toString())
-        def whenObj = createX( TaskClosure, newArgs )
-
-        // creates a method call expression for the method `when`
-        def method = new MethodCallExpression(VariableExpression.THIS_EXPRESSION, blockName, whenObj)
-        parent.getStatements().add(0, new ExpressionStatement(method))
-
-        return block
-    }
-
-    /**
-     * Wrap the user provided piece of code, either a script or a closure with a {@code BodyDef} object
-     *
-     * @param closure
-     * @param source
-     * @param scriptOrNative
-     * @param unit
-     * @return a {@code BodyDef} object
-     */
-    private Expression makeScriptWrapper( ClosureExpression closure, CharSequence source, String section, SourceUnit unit ) {
-
-        final List newArgs = []
-        newArgs << (closure)
-        newArgs << ( new ConstantExpression(source.toString()) )
-        newArgs << ( new ConstantExpression(section) )
-
-        // collect all variable tokens and pass them as single list argument
-        final variables = fetchVariables(closure,unit)
-        final listArg = new ArrayList(variables.size())
-        for( TokenValRef var: variables ) {
-            def pName = new ConstantExpression(var.name)
-            def pLine = new ConstantExpression(var.lineNum)
-            def pCol = new ConstantExpression(var.colNum)
-            listArg << createX( TokenValRef, pName, pLine, pCol )
-        }
-        newArgs << ( new ListExpression(listArg) )
-
-        // invokes the BodyDef constructor
-        createX( BodyDef, newArgs )
-    }
-
-    /**
-     * Read the user provided script source string
-     *
-     * @param node
-     * @param buffer
-     * @param unit
-     */
-    private void readSource( ASTNode node, StringBuilder buffer, SourceUnit unit, stripBrackets=false ) {
-        final colx = node.getColumnNumber()
-        final colz = node.getLastColumnNumber()
-        final first = node.getLineNumber()
-        final last = node.getLastLineNumber()
-        for( int i=first; i<=last; i++ ) {
-            def line = unit.source.getLine(i, null)
-            if( i==last ) {
-                line = line.substring(0,colz-1)
-                if( stripBrackets ) {
-                    line = line.replaceFirst(/}.*$/,'')
-                    if( !line.trim() ) continue
-                }
-            }
-            if( i==first ) {
-                line = line.substring(colx-1)
-                if( stripBrackets ) {
-                    line = line.replaceFirst(/^.*\{/,'').trim()
-                    if( !line.trim() ) continue
-                }
-            }
-            buffer.append(line) .append('\n')
-        }
-    }
-
-    protected void fixLazyGString( Statement stm ) {
-        if( stm instanceof ExpressionStatement && stm.getExpression() instanceof MethodCallExpression ) {
-            new GStringToLazyVisitor(unit).visitExpressionStatement(stm)
-        }
-    }
-
-    protected void fixDirectiveWithNegativeValue( Statement stm ) {
-        if( stm instanceof ExpressionStatement && stm.getExpression() instanceof BinaryExpression ) {
-            def binary = (BinaryExpression)stm.getExpression()
-            if(!(binary.leftExpression instanceof VariableExpression))
-                return
-            if( binary.operation.type != Types.MINUS )
-                return
-
-            // -- transform the binary expression into a method call expression
-            //    where the left expression represents the method name to invoke
-            def methodName = ((VariableExpression)binary.leftExpression).name
-
-            // -- wrap the value into a minus operator
-            def value = (Expression)new UnaryMinusExpression( binary.rightExpression )
-            def args = new ArgumentListExpression( [value] )
-
-            // -- create the method call expression and replace it to the binary expression
-            def call = new MethodCallExpression(new VariableExpression('this'), methodName, args)
-            stm.setExpression(call)
-
-        }
-    }
-
-    protected void fixStdinStdout( ExpressionStatement stm ) {
-
-        // transform the following syntax:
-        //      `stdin from x`  --> stdin() from (x)
-        //      `stdout into x` --> `stdout() into (x)`
-        VariableExpression varX
-        if( stm.expression instanceof PropertyExpression ) {
-            def expr = (PropertyExpression)stm.expression
-            def obj = expr.objectExpression
-            def prop = expr.property as ConstantExpression
-            def target = new VariableExpression(prop.text)
-
-            if( obj instanceof MethodCallExpression ) {
-                def methodCall = obj as MethodCallExpression
-                if( 'stdout' == methodCall.getMethodAsString() ) {
-                    def stdout = new MethodCallExpression( new VariableExpression('this'), 'stdout', new ArgumentListExpression()  )
-                    def into = new MethodCallExpression(stdout, 'into', new ArgumentListExpression(target))
-                    // remove replace the old one with the new one
-                    stm.setExpression( into )
-                }
-                else if( 'stdin' == methodCall.getMethodAsString() ) {
-                    def stdin = new MethodCallExpression( new VariableExpression('this'), 'stdin', new ArgumentListExpression()  )
-                    def from = new MethodCallExpression(stdin, 'from', new ArgumentListExpression(target))
-                    // remove replace the old one with the new one
-                    stm.setExpression( from )
-                }
-            }
-        }
-        // transform the following syntax:
-        //      `stdout into (x,y,..)` --> `stdout() into (x,y,..)`
-        else if( stm.expression instanceof MethodCallExpression ) {
-            def methodCall = (MethodCallExpression)stm.expression
-            if( 'stdout' == methodCall.getMethodAsString() ) {
-                def args = methodCall.getArguments()
-                if( args instanceof ArgumentListExpression && args.getExpressions() && args.getExpression(0) instanceof MethodCallExpression ) {
-                    def methodCall2 = (MethodCallExpression)args.getExpression(0)
-                    def args2 = methodCall2.getArguments()
-                    if( args2 instanceof ArgumentListExpression && methodCall2.methodAsString == 'into') {
-                        def vars = args2.getExpressions()
-                        def stdout = new MethodCallExpression( new VariableExpression('this'), 'stdout', new ArgumentListExpression()  )
-                        def into = new MethodCallExpression(stdout, 'into', new ArgumentListExpression(vars))
-                        // remove replace the old one with the new one
-                        stm.setExpression( into )
-                    }
-                }
-            }
-        }
-        else if( (varX=isVariableX(stm.expression)) && (varX.name=='stdin' || varX.name=='stdout') && NF.isDsl2() ) {
-            final name = varX.name=='stdin' ? '_in_stdin' : '_out_stdout'
-            final call = new MethodCallExpression( new VariableExpression('this'), name, new ArgumentListExpression()  )
-            // remove replace the old one with the new one
-            stm.setExpression(call)
-        }
-    }
-
-    /*
-     * handle *input* parameters
-     */
-    protected void convertInputMethod( Expression expression ) {
-        log.trace "convert > input expression: $expression"
-
-        if( expression instanceof MethodCallExpression ) {
-
-            def methodCall = expression as MethodCallExpression
-            def methodName = methodCall.getMethodAsString()
-            def nested = methodCall.objectExpression instanceof MethodCallExpression
-            log.trace "convert > input method: $methodName"
-
-            if( methodName in ['val','env','file','each','set','stdin','path','tuple'] ) {
-                //this methods require a special prefix
-                if( !nested )
-                    methodCall.setMethod( new ConstantExpression('_in_' + methodName) )
-
-                fixMethodCall(methodCall)
-            }
-
-            /*
-             * Handles a GString a file name, like this:
-             *
-             *      input:
-             *        file x name "$var_name" from q
-             *
-             */
-            else if( methodName == 'name' && isWithinMethod(expression, 'file') ) {
-                varToConstX(methodCall.getArguments())
-            }
-
-            // invoke on the next method call
-            if( expression.objectExpression instanceof MethodCallExpression ) {
-                convertInputMethod(methodCall.objectExpression)
-            }
-        }
-
-        else if( expression instanceof PropertyExpression ) {
-            // invoke on the next method call
-            if( expression.objectExpression instanceof MethodCallExpression ) {
-                convertInputMethod(expression.objectExpression)
-            }
-        }
-
-    }
-
-    protected boolean isWithinMethod(MethodCallExpression method, String name) {
-        if( method.objectExpression instanceof MethodCallExpression ) {
-            return isWithinMethod(method.objectExpression as MethodCallExpression, name)
-        }
-
-        return method.getMethodAsString() == name
-    }
-
-    /**
-     * Transform a map entry `emit: something` into `emit: 'something'
-     * (ie. as a constant) in a map expression passed as argument to
-     * a method call. This allow the syntax
-     *
-     *   output:
-     *   path 'foo', emit: bar
-     *
-     * @param call
-     */
-    protected void fixOutEmitOption(MethodCallExpression call) {
-        List args = isTupleX(call.arguments)?.expressions
-        if( !args ) return
-        if( args.size()<2 && (args.size()!=1 || call.methodAsString!='_out_stdout')) return
-        MapExpression map = isMapX(args[0])
-        if( !map ) return
-        for( int i=0; i output expression: $expression"
-
-        if( !(expression instanceof MethodCallExpression) ) {
-            return
-        }
-
-        def methodCall = expression as MethodCallExpression
-        def methodName = methodCall.getMethodAsString()
-        def nested = methodCall.objectExpression instanceof MethodCallExpression
-        log.trace "convert > output method: $methodName"
-
-        if( methodName in ['val','env','file','set','stdout','path','tuple'] && !nested ) {
-            // prefix the method name with the string '_out_'
-            methodCall.setMethod( new ConstantExpression('_out_' + methodName) )
-            fixMethodCall(methodCall)
-            fixOutEmitOption(methodCall)
-        }
-
-        else if( methodName in ['into','mode'] ) {
-            fixMethodCall(methodCall)
-        }
-
-        // continue to traverse
-        if( methodCall.objectExpression instanceof MethodCallExpression ) {
-            convertOutputMethod(methodCall.objectExpression)
-        }
-
-    }
-
-    private boolean withinTupleMethod
-
-    private boolean withinEachMethod
-
-    /**
-     * This method converts the a method call argument from a Variable to a Constant value
-     * so that it is possible to reference variable that not yet exist
-     *
-     * @param methodCall The method object for which it is required to change args definition
-     * @param flagVariable Whenever append a flag specified if the variable replacement has been applied
-     * @param index The index of the argument to modify
-     * @return
-     */
-    protected void fixMethodCall( MethodCallExpression methodCall ) {
-        final name = methodCall.methodAsString
-
-        withinTupleMethod = name == '_in_set' || name == '_out_set' || name == '_in_tuple' || name == '_out_tuple'
-        withinEachMethod = name == '_in_each'
-
-        try {
-            if( isOutputWithPropertyExpression(methodCall) ) {
-                // transform an output value declaration such
-                //   output: val( obj.foo )
-                // to
-                //   output: val({ obj.foo })
-                wrapPropertyToClosure((ArgumentListExpression)methodCall.getArguments())
-            }
-            else
-                varToConstX(methodCall.getArguments())
-
-        } finally {
-            withinTupleMethod = false
-            withinEachMethod = false
-        }
-    }
-
-    static final private List OUT_PROPERTY_VALID_TYPES = ['_out_val', '_out_env', '_out_file', '_out_path']
-
-    protected boolean isOutputWithPropertyExpression(MethodCallExpression methodCall) {
-        if( methodCall.methodAsString !in OUT_PROPERTY_VALID_TYPES  )
-            return false
-        if( methodCall.getArguments() instanceof ArgumentListExpression ) {
-            def args = (ArgumentListExpression)methodCall.getArguments()
-            if( args.size()==0 || args.size()>2 )
-                return false
-
-            return args.last() instanceof PropertyExpression
-        }
-
-        return false
-    }
-
-    protected void wrapPropertyToClosure(ArgumentListExpression expr) {
-        final args = expr as ArgumentListExpression
-        final property = (PropertyExpression) args.last()
-        final closure = wrapPropertyToClosure(property)
-        args.getExpressions().set(args.size()-1, closure)
-    }
-
-    protected ClosureExpression wrapPropertyToClosure(PropertyExpression property)  {
-        def block = new BlockStatement()
-        block.addStatement( new ExpressionStatement(property) )
-
-        def closure = new ClosureExpression( Parameter.EMPTY_ARRAY, block )
-        closure.variableScope = new VariableScope(block.variableScope)
-
-        return closure
-    }
-
-
-    protected Expression varToStrX( Expression expr ) {
-        if( expr instanceof VariableExpression ) {
-            def name = ((VariableExpression) expr).getName()
-            return createX( LazyVar, new ConstantExpression(name) )
-        }
-        else if( expr instanceof PropertyExpression ) {
-            // transform an output declaration such
-            // output: tuple val( obj.foo )
-            //  to
-            // output: tuple val({ obj.foo })
-            return wrapPropertyToClosure(expr)
-        }
-
-        if( expr instanceof TupleExpression )  {
-            def i = 0
-            def list = expr.getExpressions()
-            for( Expression item : list ) {
-                list[i++] = varToStrX(item)
-            }
-
-            return expr
-        }
-
-        return expr
-    }
-
-    protected Expression varToConstX( Expression expr ) {
-
-        if( expr instanceof VariableExpression ) {
-            // when it is a variable expression, replace it with a constant representing
-            // the variable name
-            def name = ((VariableExpression) expr).getName()
-
-            /*
-             * the 'stdin' is used as placeholder for the standard input in the tuple definition. For example:
-             *
-             * input:
-             *    tuple( stdin, .. ) from q
-             */
-            if( name == 'stdin' && withinTupleMethod )
-                return createX( TokenStdinCall )
-
-            /*
-             * input:
-             *    tuple( stdout, .. )
-             */
-            else if ( name == 'stdout' && withinTupleMethod )
-                return createX( TokenStdoutCall )
-
-            else
-                return createX( LazyVar, new ConstantExpression(name) )
-        }
-
-        if( expr instanceof MethodCallExpression ) {
-            def methodCall = expr as MethodCallExpression
-
-            /*
-             * replace 'file' method call in the tuple definition, for example:
-             *
-             * input:
-             *   tuple( file(fasta:'*.fa'), .. ) from q
-             */
-            if( methodCall.methodAsString == 'file' && (withinTupleMethod || withinEachMethod) ) {
-                def args = (TupleExpression) varToConstX(methodCall.arguments)
-                return createX( TokenFileCall, args )
-            }
-            else if( methodCall.methodAsString == 'path' && (withinTupleMethod || withinEachMethod) ) {
-                def args = (TupleExpression) varToConstX(methodCall.arguments)
-                return createX( TokenPathCall, args )
-            }
-
-            /*
-             * input:
-             *  tuple( env(VAR_NAME) ) from q
-             */
-            if( methodCall.methodAsString == 'env' && withinTupleMethod ) {
-                def args = (TupleExpression) varToStrX(methodCall.arguments)
-                return createX( TokenEnvCall, args )
-            }
-
-            /*
-             * input:
-             *   tuple val(x), .. from q
-             */
-            if( methodCall.methodAsString == 'val' && withinTupleMethod ) {
-                def args = (TupleExpression) varToStrX(methodCall.arguments)
-                return createX( TokenValCall, args )
-            }
-
-        }
-
-        // -- TupleExpression or ArgumentListExpression
-        if( expr instanceof TupleExpression )  {
-            def i = 0
-            def list = expr.getExpressions()
-            for( Expression item : list )  {
-                list[i++] = varToConstX(item)
-            }
-            return expr
-        }
-
-        return expr
-    }
-
-    /**
-     * Wrap a generic expression with in a closure expression
-     *
-     * @param block The block to which the resulting closure has to be appended
-     * @param expr The expression to the wrapped in a closure
-     * @param len
-     * @return A tuple in which:
-     *      
  • 1st item: {@code true} if successful or {@code false} otherwise
-     *      
  • 2nd item: on error condition the line containing the error in the source script, zero otherwise
-     *      
  • 3rd item: on error condition the column containing the error in the source script, zero otherwise
-     *
-     */
-    protected boolean wrapExpressionWithClosure( BlockStatement block, Expression expr, int len, CharSequence source, SourceUnit unit ) {
-        if( expr instanceof GStringExpression || expr instanceof ConstantExpression ) {
-            // remove the last expression
-            block.statements.remove(len-1)
-
-            // and replace it by a wrapping closure
-            def closureExp = new ClosureExpression( Parameter.EMPTY_ARRAY, new ExpressionStatement(expr) )
-            closureExp.variableScope = new VariableScope(block.variableScope)
-
-            // append to the list of statement
-            //def wrap = newObj(BodyDef, closureExp, new ConstantExpression(source.toString()), ConstantExpression.TRUE)
-            def wrap = makeScriptWrapper(closureExp, source, 'script', unit )
-            block.statements.add( new ExpressionStatement(wrap) )
-
-            return true
-        }
-        else if( expr instanceof ClosureExpression ) {
-            // do not touch it
-            return true
-        }
-        else {
-            log.trace "Invalid process result expression: ${expr} -- Only constant or string expression can be used"
-        }
-
-        return false
-    }
-
-    protected boolean isIllegalName(String name, ASTNode node) {
-        if( name in RESERVED_NAMES ) {
-            unit.addError( new SyntaxException("Identifier `$name` is reserved for internal use", node.lineNumber, node.columnNumber+8) )
-            return true
-        }
-        if( name in workflowNames || name in processNames ) {
-            unit.addError( new SyntaxException("Identifier `$name` is already used by another definition", node.lineNumber, node.columnNumber+8) )
-            return true
-        }
-        if( name.contains(SCOPE_SEP) ) {
-            def offset =  8+2+ name.indexOf(SCOPE_SEP)
-            unit.addError( new SyntaxException("Process and workflow names cannot contain colon character", node.lineNumber, node.columnNumber+offset) )
-            return true
-        }
-        return false
-    }
-
-    /**
-     * This method handle the process definition, so that it transform the user entered syntax
-     *    process myName ( named: args, ..  ) { code .. }
-     *
-     * into
-     *    process ( [named:args,..], String myName )  { }
-     *
-     * @param methodCall
-     * @param unit
-     */
-    protected void convertProcessDef( MethodCallExpression methodCall, SourceUnit unit ) {
-        log.trace "Converts 'process' ${methodCall.arguments}"
-
-        assert methodCall.arguments instanceof ArgumentListExpression
-        def list = (methodCall.arguments as ArgumentListExpression).getExpressions()
-
-        // extract the first argument which has to be a method-call expression
-        // the name of this method represent the *process* name
-        if( list.size() != 1 || !list[0].class.isAssignableFrom(MethodCallExpression) ) {
-            log.debug "Missing name in process definition at line: ${methodCall.lineNumber}"
-            unit.addError( new SyntaxException("Process definition syntax error -- A string identifier must be provided after the `process` keyword", methodCall.lineNumber, methodCall.columnNumber+7))
-            return
-        }
-
-        def nested = list[0] as MethodCallExpression
-        def name = nested.getMethodAsString()
-        // check the process name is not defined yet
-        if( isIllegalName(name, methodCall) ) {
-            return
-        }
-        processNames.add(name)
-
-        // the nested method arguments are the arguments to be passed
-        // to the process definition, plus adding the process *name*
-        // as an extra item in the arguments list
-        def args = nested.getArguments() as ArgumentListExpression
-        log.trace "Process name: $name with args: $args"
-
-        // make sure to add the 'name' after the map item
-        // (which represent the named parameter attributes)
-        list = args.getExpressions()
-        if( list.size()>0 && list[0] instanceof MapExpression ) {
-            list.add(1, new ConstantExpression(name))
-        }
-        else {
-            list.add(0, new ConstantExpression(name))
-        }
-
-        // set the new list as the new arguments
-        methodCall.setArguments( args )
-
-        // now continue as before !
-        convertProcessBlock(methodCall, unit)
-    }
-
-    /**
-     * Fetch all the variable references in a closure expression.
-     *
-     * @param closure
-     * @param unit
-     * @return The set of variable names referenced in the script. NOTE: it includes properties in the form {@code object.propertyName}
-     */
-    protected Set fetchVariables( ClosureExpression closure, SourceUnit unit ) {
-        def visitor = new VariableVisitor(unit)
-        visitor.visitClosureExpression(closure)
-        return visitor.allVariables
-    }
-
-}
diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/ProcessFn.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSL.groovy
similarity index 66%
rename from modules/nextflow/src/main/groovy/nextflow/ast/ProcessFn.groovy
rename to modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSL.groovy
index 11ce865f49..7d54cb0e8f 100644
--- a/modules/nextflow/src/main/groovy/nextflow/ast/ProcessFn.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSL.groovy
@@ -21,23 +21,12 @@ import java.lang.annotation.Retention
 import java.lang.annotation.RetentionPolicy
 import java.lang.annotation.Target
 
+import org.codehaus.groovy.transform.GroovyASTTransformationClass
+
 /**
- * Annotation for process functions.
- *
- * @author Ben Sherman 
+ * Marker interface which to apply AST transformation to {@code process} declaration
  */
-@Retention(RetentionPolicy.RUNTIME)
+@Retention(RetentionPolicy.SOURCE)
 @Target(ElementType.METHOD)
-@interface ProcessFn {
-    Class directives() default {->}
-    Class inputs() default {->}
-    Class outputs() default {->}
-
-    boolean script() default false
-    boolean shell() default false
-
-    // injected via AST transform
-    String[] params()
-    String source()
-    String[] vars()
-}
+@GroovyASTTransformationClass(classes = [NextflowDSLImpl])
+@interface NextflowDSL {}
diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSLImpl.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSLImpl.groovy
new file mode 100644
index 0000000000..ef1b36a573
--- /dev/null
+++ b/modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSLImpl.groovy
@@ -0,0 +1,1272 @@
+/*
+ * Copyright 2013-2023, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package nextflow.ast
+
+import static nextflow.Const.*
+import static nextflow.ast.ASTHelpers.*
+import static org.codehaus.groovy.ast.tools.GeneralUtils.*
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import nextflow.NF
+import nextflow.script.BaseScript
+import nextflow.script.BodyDef
+import nextflow.script.IncludeDef
+import nextflow.script.TaskClosure
+import nextflow.script.TokenEnvCall
+import nextflow.script.TokenFileCall
+import nextflow.script.TokenPathCall
+import nextflow.script.TokenStdinCall
+import nextflow.script.TokenStdoutCall
+import nextflow.script.TokenValCall
+import nextflow.script.TokenValRef
+import nextflow.util.LazyVar
+import org.codehaus.groovy.ast.ASTNode
+import org.codehaus.groovy.ast.ClassCodeVisitorSupport
+import org.codehaus.groovy.ast.ClassNode
+import org.codehaus.groovy.ast.MethodNode
+import org.codehaus.groovy.ast.Parameter
+import org.codehaus.groovy.ast.VariableScope
+import org.codehaus.groovy.ast.expr.ArgumentListExpression
+import org.codehaus.groovy.ast.expr.BinaryExpression
+import org.codehaus.groovy.ast.expr.CastExpression
+import org.codehaus.groovy.ast.expr.ClosureExpression
+import org.codehaus.groovy.ast.expr.ConstantExpression
+import org.codehaus.groovy.ast.expr.Expression
+import org.codehaus.groovy.ast.expr.GStringExpression
+import org.codehaus.groovy.ast.expr.ListExpression
+import org.codehaus.groovy.ast.expr.MapEntryExpression
+import org.codehaus.groovy.ast.expr.MapExpression
+import org.codehaus.groovy.ast.expr.MethodCallExpression
+import org.codehaus.groovy.ast.expr.PropertyExpression
+import org.codehaus.groovy.ast.expr.TupleExpression
+import org.codehaus.groovy.ast.expr.UnaryMinusExpression
+import org.codehaus.groovy.ast.expr.VariableExpression
+import org.codehaus.groovy.ast.stmt.BlockStatement
+import org.codehaus.groovy.ast.stmt.ExpressionStatement
+import org.codehaus.groovy.ast.stmt.ReturnStatement
+import org.codehaus.groovy.ast.stmt.Statement
+import org.codehaus.groovy.control.CompilePhase
+import org.codehaus.groovy.control.SourceUnit
+import org.codehaus.groovy.syntax.SyntaxException
+import org.codehaus.groovy.syntax.Token
+import org.codehaus.groovy.syntax.Types
+import org.codehaus.groovy.transform.ASTTransformation
+import org.codehaus.groovy.transform.GroovyASTTransformation
+/**
+ * Implement some syntax sugars of Nextflow DSL scripting.
+ *
+ * @author Paolo Di Tommaso 
+ */
+
+@Slf4j
+@CompileStatic
+@GroovyASTTransformation(phase = CompilePhase.CONVERSION)
+class NextflowDSLImpl implements ASTTransformation {
+
+    final static private String WORKFLOW_TAKE = 'take'
+    final static private String WORKFLOW_EMIT = 'emit'
+    final static private String WORKFLOW_MAIN = 'main'
+    final static private List SCOPES = [WORKFLOW_TAKE, WORKFLOW_EMIT, WORKFLOW_MAIN]
+
+    final static public String PROCESS_WHEN = 'when'
+    final static public String PROCESS_STUB = 'stub'
+
+    static public String OUT_PREFIX = '$out'
+
+    static private Set RESERVED_NAMES
+
+    static {
+        // method names implicitly defined by the groovy script SHELL
+        RESERVED_NAMES = ['main','run','runScript'] as Set
+        // existing method cannot be used for custom script definition
+        for( def method : BaseScript.getMethods() ) {
+            RESERVED_NAMES.add(method.name)
+        }
+
+    }
+
+    @Override
+    void visit(ASTNode[] astNodes, SourceUnit unit) {
+        createVisitor(unit).visitClass((ClassNode)astNodes[1])
+    }
+
+    /*
+     * create the code visitor
+     */
+    protected ClassCodeVisitorSupport createVisitor( SourceUnit unit ) {
+        new DslCodeVisitor(unit)
+    }
+
+    @CompileStatic
+    static class DslCodeVisitor extends ClassCodeVisitorSupport {
+
+
+        final private SourceUnit unit
+
+        private String currentTaskName
+
+        private String currentLabel
+
+        private String bodyLabel
+
+        private Set processNames = []
+
+        private Set workflowNames = []
+
+        private Set functionNames = []
+
+        private int anonymousWorkflow
+
+        protected SourceUnit getSourceUnit() { unit }
+
+
+        DslCodeVisitor(SourceUnit unit) {
+            this.unit = unit
+        }
+
+        @Override
+        void visitMethod(MethodNode node) {
+            if( node.public && !node.static && !node.synthetic && !node.metaDataMap?.'org.codehaus.groovy.ast.MethodNode.isScriptBody') {
+                if( !isIllegalName(node.name, node))
+                    functionNames.add(node.name)
+            }
+            super.visitMethod(node)
+        }
+
+        @Override
+        void visitMethodCallExpression(MethodCallExpression methodCall) {
+            // pre-condition to be verified to apply the transformation
+            final preCondition = methodCall.objectExpression?.getText() == 'this'
+            final methodName = methodCall.getMethodAsString()
+
+            /*
+             * intercept the *process* method in order to transform the script closure
+             */
+            if( methodName == 'process' && preCondition ) {
+
+                // clear block label
+                bodyLabel = null
+                currentLabel = null
+                currentTaskName = methodName
+                try {
+                    convertProcessDef(methodCall,sourceUnit)
+                    super.visitMethodCallExpression(methodCall)
+                }
+                finally {
+                    currentTaskName = null
+                }
+            }
+            else if( methodName == 'workflow' && preCondition ) {
+                convertWorkflowDef(methodCall,sourceUnit)
+                super.visitMethodCallExpression(methodCall)
+            }
+
+            // just apply the default behavior
+            else {
+                super.visitMethodCallExpression(methodCall)
+            }
+
+        }
+
+        @Override
+        void visitExpressionStatement(ExpressionStatement stm) {
+            if( stm.text.startsWith('this.include(') && stm.getExpression() instanceof MethodCallExpression )  {
+                final methodCall = (MethodCallExpression)stm.getExpression()
+                convertIncludeDef(methodCall)
+                // this is necessary to invoke the `load` method on the include definition
+                final loadCall = new MethodCallExpression(methodCall, 'load0', new ArgumentListExpression(new VariableExpression('params')))
+                stm.setExpression(loadCall)
+            }
+            super.visitExpressionStatement(stm)
+        }
+
+        protected void convertIncludeDef(MethodCallExpression call) {
+            if( call.methodAsString=='include' && call.arguments instanceof ArgumentListExpression ) {
+                final allArgs = (ArgumentListExpression)call.arguments
+                if( allArgs.size() != 1 ) {
+                    syntaxError(call, "Not a valid include definition -- it must specify the module path")
+                    return
+                }
+
+                final arg = allArgs[0]
+                final newArgs = new ArgumentListExpression()
+                if( arg instanceof ConstantExpression ) {
+                    newArgs.addExpression( createX(IncludeDef, arg) )
+                }
+                else if( arg instanceof VariableExpression ) {
+                    // the name of the component i.e. process, workflow, etc to import
+                    final component = arg.getName()
+                    // wrap the name in a `LazyVar` type
+                    final token = createX(LazyVar, new ConstantExpression(component))
+                    // create a new `IncludeDef` object
+                    newArgs.addExpression(createX(IncludeDef, token))
+                }
+                else if( arg instanceof CastExpression && arg.getExpression() instanceof VariableExpression) {
+                    def cast = (CastExpression)arg
+                    // the name of the component i.e. process, workflow, etc to import
+                    final component = (cast.expression as VariableExpression).getName()
+                    // wrap the name in a `LazyVar` type
+                    final token = createX(LazyVar, new ConstantExpression(component))
+                    // the alias to give it
+                    final alias = constX(cast.type.name)
+                    newArgs.addExpression( createX(IncludeDef, token, alias) )
+                }
+                else if( arg instanceof ClosureExpression ) {
+                    // multiple modules inclusion 
+                    final block = (BlockStatement)arg.getCode()
+                    final modulesList = new ListExpression()
+                    for( Statement stm : block.statements ) {
+                        if( stm instanceof ExpressionStatement ) {
+                            CastExpression castX
+                            VariableExpression varX
+                            Expression moduleX
+                            if( (varX=isVariableX(stm.expression)) ) {
+                                def name = constX(varX.name)
+                                moduleX = createX(IncludeDef.Module, name)
+                            }
+                            else if( (castX=isCastX(stm.expression)) && (varX=isVariableX(castX.expression)) ) {
+                                def name = constX(varX.name)
+                                final alias = constX(castX.type.name)
+                                moduleX = createX(IncludeDef.Module, name, alias)
+                            }
+                            else {
+                                syntaxError(call, "Not a valid include module name")
+                                return
+                            }
+                            modulesList.addExpression(moduleX)
+                        }
+                        else {
+                            syntaxError(call, "Not a valid include module name")
+                            return
+                        }
+
+                    }
+                    newArgs.addExpression( createX(IncludeDef, modulesList) )
+                }
+                else {
+                    syntaxError(call, "Not a valid include definition -- it must specify the module path as a string")
+                    return
+                }
+                call.setArguments(newArgs)
+            }
+            else if( call.objectExpression instanceof MethodCallExpression ) {
+                convertIncludeDef((MethodCallExpression)call.objectExpression)
+            }
+        }
+
+        /*
+         * this method transforms the DSL definition
+         *
+         *   workflow foo {
+         *     code
+         *   }
+         *
+         * into a method invocation as
+         *
+         *   workflow('foo', { -> code })
+         *
+         */
+        protected void convertWorkflowDef(MethodCallExpression methodCall, SourceUnit unit) {
+            log.trace "Convert 'workflow' ${methodCall.arguments}"
+
+            assert methodCall.arguments instanceof ArgumentListExpression
+            def args = (ArgumentListExpression)methodCall.arguments
+            def len = args.size()
+
+            // anonymous workflow definition
+            if( len == 1 && args[0] instanceof ClosureExpression ) {
+                if( anonymousWorkflow++ > 0 ) {
+                    unit.addError( new SyntaxException("Duplicate entry workflow definition", methodCall.lineNumber, methodCall.columnNumber+8))
+                    return
+                }
+
+                def newArgs = new ArgumentListExpression()
+                def body = (ClosureExpression)args[0]
+                newArgs.addExpression( makeWorkflowDefWrapper(body,true) )
+                methodCall.setArguments( newArgs )
+                return 
+            }
+
+            // extract the first argument which has to be a method-call expression
+            // the name of this method represent the *workflow* name
+            if( len != 1 || !args[0].class.isAssignableFrom(MethodCallExpression) ) {
+                log.debug "Missing name in workflow definition at line: ${methodCall.lineNumber}"
+                unit.addError( new SyntaxException("Workflow definition syntax error -- A string identifier must be provided after the `workflow` keyword", methodCall.lineNumber, methodCall.columnNumber+8))
+                return
+            }
+
+            final nested = args[0] as MethodCallExpression
+            final name = nested.getMethodAsString()
+            // check the process name is not defined yet
+            if( isIllegalName(name, methodCall) ) {
+                return
+            }
+            workflowNames.add(name)
+
+            // the nested method arguments are the arguments to be passed
+            // to the process definition, plus adding the process *name*
+            // as an extra item in the arguments list
+            args = (ArgumentListExpression)nested.getArguments()
+            len = args.size()
+            log.trace "Workflow name: $name with args: $args"
+
+            // make sure to add the 'name' after the map item
+            // (which represent the named parameter attributes)
+            def newArgs = new ArgumentListExpression()
+
+            // add the workflow body def
+            if( len != 1 || !(args[0] instanceof ClosureExpression)) {
+                syntaxError(methodCall, "Invalid workflow definition")
+                return
+            }
+
+            final body = (ClosureExpression)args[0]
+            newArgs.addExpression( constX(name) )
+            newArgs.addExpression( makeWorkflowDefWrapper(body,false) )
+
+            // set the new list as the new arguments
+            methodCall.setArguments( newArgs )
+        }
+
+
+        protected Statement normWorkflowParam(ExpressionStatement stat, String type, Set uniqueNames, List body) {
+            MethodCallExpression callx
+            VariableExpression varx
+
+            if( (callx=isMethodCallX(stat.expression)) && isThisX(callx.objectExpression) ) {
+                final name = "_${type}_${callx.methodAsString}"
+                return stmt( callThisX(name, callx.arguments) )
+            }
+
+            if( (varx=isVariableX(stat.expression)) ) {
+                final name = "_${type}_${varx.name}"
+                return stmt( callThisX(name) )
+            }
+
+            if( type == WORKFLOW_EMIT ) {
+                return createAssignX(stat, body, type, uniqueNames)
+            }
+
+            syntaxError(stat, "Workflow malformed parameter definition")
+            return stat
+        }
+
+        protected Statement createAssignX(ExpressionStatement stat, List body, String type, Set uniqueNames) {
+            BinaryExpression binx
+            MethodCallExpression callx
+            Expression args=null
+
+            if( (binx=isAssignX(stat.expression)) ) {
+                // keep the statement in body to allow it to be evaluated
+                body.add(stat)
+                // and create method call expr to capture the var name in the emission
+                final left = (VariableExpression)binx.leftExpression
+                final name = "_${type}_${left.name}"
+                return stmt( callThisX(name) )
+            }
+
+            if( (callx=isMethodCallX(stat.expression)) && callx.objectExpression.text!='this' && hasTo(callx)) {
+                // keep the args
+                args = callx.arguments
+                // replace the method call expression with a property
+                stat.expression = new PropertyExpression(callx.objectExpression, callx.method)
+                // then, fallback to default case
+            }
+
+            // wrap the expression into a assignment expression
+            final var = getNextName(uniqueNames)
+            final left = new VariableExpression(var)
+            final right = stat.expression
+            final token = new Token(Types.ASSIGN, '=', -1, -1)
+            final assign = new BinaryExpression(left, token, right)
+            body.add(stmt(assign))
+
+            // the call method statement for the emit declaration
+            final name="_${type}_${var}"
+            callx =  args ? callThisX(name, args) : callThisX(name)
+            return stmt(callx)
+        }
+
+        protected boolean hasTo(MethodCallExpression callX) {
+            def tupleX = isTupleX(callX.arguments)
+            if( !tupleX ) return false
+            if( !tupleX.expressions ) return false
+            def mapX = isMapX(tupleX.expressions[0])
+            if( !mapX ) return false
+            def entry = mapX.getMapEntryExpressions().find { isConstX(it.keyExpression).text=='to' }
+            return entry != null
+        }
+
+        protected String getNextName(Set allNames) {
+            String result
+            while( true ) {
+                result = OUT_PREFIX + allNames.size()
+                if( allNames.add(result) )
+                    break
+            }
+            return result
+        }
+
+        protected Expression makeWorkflowDefWrapper( ClosureExpression closure, boolean anonymous ) {
+
+            final codeBlock = (BlockStatement) closure.code
+            final codeStms = codeBlock.statements
+            final scope = codeBlock.variableScope
+
+            final visited = new HashMap(5);
+            final emitNames = new LinkedHashSet(codeStms.size())
+            final wrap = new ArrayList(codeStms.size())
+            final body = new ArrayList(codeStms.size())
+            final source = new StringBuilder()
+            String context = null
+            String previous = null
+            for( Statement stm : codeStms ) {
+                previous = context
+                context = stm.statementLabel ?: context
+                // check for changing context
+                if( context && context != previous ) {
+                    if( visited[context] && visited[previous] ) {
+                        syntaxError(stm, "Unexpected workflow `${context}` context here")
+                        break
+                    }
+                }
+                visited[context] = true
+
+                switch (context) {
+                    case WORKFLOW_TAKE:
+                    case WORKFLOW_EMIT:
+                        if( !(stm instanceof ExpressionStatement) ) {
+                            syntaxError(stm, "Workflow malformed parameter definition")
+                            break
+                        }
+                        wrap.add(normWorkflowParam(stm as ExpressionStatement, context, emitNames, body))
+                    break
+
+                    case WORKFLOW_MAIN:
+                        body.add(stm)
+                        break
+
+                    default:
+                        if( context ) {
+                            def opts = SCOPES.closest(context)
+                            def msg = "Unknown execution scope '$context:'"
+                            if( opts ) msg += " -- Did you mean ${opts.collect{"'$it'"}.join(', ')}"
+                            syntaxError(stm, msg)
+                        }
+                        body.add(stm)
+                }
+            }
+            // read the closure source
+            readSource(closure, source, unit, true)
+
+            final bodyClosure = closureX(null, block(scope, body))
+            final invokeBody = makeScriptWrapper(bodyClosure, source.toString(), 'workflow', unit)
+            wrap.add( stmt(invokeBody) )
+
+            closureX(null, block(scope, wrap))
+        }
+
+        protected void syntaxError(ASTNode node, String message) {
+            int line = node.lineNumber
+            int coln = node.columnNumber
+            unit.addError( new SyntaxException(message,line,coln))
+        }
+
+        /**
+         * Transform a DSL `process` definition into a proper method invocation
+         *
+         * @param methodCall
+         * @param unit
+         */
+        protected void convertProcessBlock( MethodCallExpression methodCall, SourceUnit unit ) {
+            log.trace "Apply task closure transformation to method call: $methodCall"
+
+            final args = methodCall.arguments as ArgumentListExpression
+            final lastArg = args.expressions.size()>0 ? args.getExpression(args.expressions.size()-1) : null
+            final isClosure = lastArg instanceof ClosureExpression
+
+            if( isClosure ) {
+                // the block holding all the statements defined in the process (closure) definition
+                final block = (lastArg as ClosureExpression).code as BlockStatement
+
+                /*
+                 * iterate over the list of statements to:
+                 * - converts the method after the 'input:' label as input parameters
+                 * - converts the method after the 'output:' label as output parameters
+                 * - collect all the statement after the 'exec:' label
+                 */
+                def source = new StringBuilder()
+                List execStatements = []
+
+                List whenStatements = []
+                def whenSource = new StringBuilder()
+
+                List stubStatements = []
+                def stubSource = new StringBuilder()
+
+
+                def iterator = block.getStatements().iterator()
+                while( iterator.hasNext() ) {
+
+                    // get next statement
+                    Statement stm = iterator.next()
+
+                    // keep track of current block label
+                    currentLabel = stm.statementLabel ?: currentLabel
+
+                    switch(currentLabel) {
+                        case 'input':
+                            if( stm instanceof ExpressionStatement ) {
+                                fixLazyGString( stm )
+                                fixStdinStdout( stm )
+                                convertInputMethod( stm.getExpression() )
+                            }
+                            break
+
+                        case 'output':
+                            if( stm instanceof ExpressionStatement ) {
+                                fixLazyGString( stm )
+                                fixStdinStdout( stm )
+                                convertOutputMethod( stm.getExpression() )
+                            }
+                            break
+
+                        case 'exec':
+                            bodyLabel = currentLabel
+                            iterator.remove()
+                            execStatements << stm
+                            readSource(stm,source,unit)
+                            break
+
+                        case 'script':
+                        case 'shell':
+                            bodyLabel = currentLabel
+                            iterator.remove()
+                            execStatements << stm
+                            readSource(stm,source,unit)
+                            break
+
+                        case PROCESS_STUB:
+                            iterator.remove()
+                            stubStatements << stm
+                            readSource(stm,stubSource,unit)
+                            break
+
+                        // capture the statements in a when guard and remove from the current block
+                        case PROCESS_WHEN:
+                            if( iterator.hasNext() ) {
+                                iterator.remove()
+                                whenStatements << stm
+                                readSource(stm,whenSource,unit)
+                                break
+                            }
+                            // when entering in this branch means that this is the last statement,
+                            // which is supposed to be the task command
+                            // hence if no previous `when` statement has been processed, a syntax error is returned
+                            else if( !whenStatements ) {
+                                int line = methodCall.lineNumber
+                                int coln = methodCall.columnNumber
+                                unit.addError(new SyntaxException("Invalid process definition -- Empty `when` or missing `script` statement", line, coln))
+                                return
+                            }
+                            else
+                                break
+
+                        default:
+                            if(currentLabel) {
+                                def line = stm.getLineNumber()
+                                def coln = stm.getColumnNumber()
+                                unit.addError(new SyntaxException("Invalid process definition -- Unknown keyword `$currentLabel`",line,coln))
+                                return
+                            }
+
+                            fixLazyGString(stm)
+                            fixDirectiveWithNegativeValue(stm)  // Fixes #180
+                    }
+                }
+
+                /*
+                 * add the `when` block if found
+                 */
+                if( whenStatements ) {
+                    addWhenGuardCall(whenStatements, whenSource, block)
+                }
+
+                /*
+                 * add try `stub` block if found
+                 */
+                if( stubStatements ) {
+                    final newBLock = addStubCall(stubStatements, stubSource, block)
+                    newBLock.visit(new TaskCmdXformVisitor(unit))
+                }
+
+                /*
+                 * wrap all the statements after the 'exec:'  label by a new closure containing them (in a new block)
+                 */
+                final len = block.statements.size()
+                boolean done = false
+                if( execStatements ) {
+                    // create a new Closure
+                    def execBlock = new BlockStatement(execStatements, new VariableScope(block.variableScope))
+                    def execClosure = new ClosureExpression( Parameter.EMPTY_ARRAY, execBlock )
+
+                    // append the new block to the
+                    // set the 'script' flag parameter
+                    def wrap = makeScriptWrapper(execClosure, source, bodyLabel, unit)
+                    block.addStatement( new ExpressionStatement(wrap) )
+                    if( bodyLabel == 'script' )
+                        block.visit(new TaskCmdXformVisitor(unit))
+                    done = true
+
+                }
+                // when only the `stub` block is defined add an empty command
+                else if ( !bodyLabel && stubStatements ) {
+                    final cmd = 'true'
+                    final list = new ArrayList(1);
+                    list.add( new ExpressionStatement(constX(cmd)) )
+                    final dummyBlock = new BlockStatement( list, new VariableScope(block.variableScope))
+                    final dummyClosure = new ClosureExpression( Parameter.EMPTY_ARRAY, dummyBlock )
+
+                    // append the new block to the
+                    // set the 'script' flag parameter
+                    final wrap = makeScriptWrapper(dummyClosure, cmd, 'script', unit)
+                    block.addStatement( new ExpressionStatement(wrap) )
+                    done = true
+                }
+
+                /*
+                 * when the last statement is a string script, the 'script:' label can be omitted
+                 */
+                else if( len ) {
+                    def stm = block.getStatements().get(len-1)
+                    readSource(stm,source,unit)
+
+                    if ( stm instanceof ReturnStatement  ){
+                        done = wrapExpressionWithClosure(block, stm.getExpression(), len, source, unit)
+                    }
+
+                    else if ( stm instanceof ExpressionStatement )  {
+                        done = wrapExpressionWithClosure(block, stm.getExpression(), len, source, unit)
+                    }
+
+                    // apply command variables escape
+                    stm.visit(new TaskCmdXformVisitor(unit))
+                }
+
+                if (!done) {
+                    log.trace "Invalid 'process' definition -- Process must terminate with string expression"
+                    int line = methodCall.lineNumber
+                    int coln = methodCall.columnNumber
+                    unit.addError( new SyntaxException("Invalid process definition -- Make sure the process ends with a script wrapped by quote characters",line,coln))
+                }
+            }
+        }
+
+        /**
+         * Converts a `when` block into a when method call expression. The when code is converted into a
+         * closure expression and set a `when` directive in the process configuration properties.
+         *
+         * See {@link nextflow.script.ProcessConfig#configProperties}
+         * See {@link nextflow.processor.TaskConfig#getGuard(java.lang.String)}
+         */
+        protected BlockStatement addWhenGuardCall( List statements, StringBuilder source, BlockStatement parent ) {
+            createBlock0(PROCESS_WHEN, statements, source, parent)
+        }
+
+        protected BlockStatement addStubCall(List statements, StringBuilder source, BlockStatement parent ) {
+            createBlock0(PROCESS_STUB, statements, source, parent)
+        }
+
+        protected BlockStatement createBlock0( String blockName, List statements, StringBuilder source, BlockStatement parent ) {
+            // wrap the code block into a closure expression
+            def block = new BlockStatement(statements, new VariableScope(parent.variableScope))
+            def closure = new ClosureExpression( Parameter.EMPTY_ARRAY, block )
+
+            // the closure expression is wrapped itself into a TaskClosure object
+            // in order to capture the closure source other than the closure code
+            List newArgs = []
+            newArgs << closure
+            newArgs << new ConstantExpression(source.toString())
+            def whenObj = createX( TaskClosure, newArgs )
+
+            // creates a method call expression for the method `when`
+            def method = new MethodCallExpression(VariableExpression.THIS_EXPRESSION, blockName, whenObj)
+            parent.getStatements().add(0, new ExpressionStatement(method))
+
+            return block
+        }
+
+        /**
+         * Wrap the user provided piece of code, either a script or a closure with a {@code BodyDef} object
+         *
+         * @param closure
+         * @param source
+         * @param scriptOrNative
+         * @param unit
+         * @return a {@code BodyDef} object
+         */
+        private Expression makeScriptWrapper( ClosureExpression closure, CharSequence source, String section, SourceUnit unit ) {
+
+            final List newArgs = []
+            newArgs << (closure)
+            newArgs << ( new ConstantExpression(source.toString()) )
+            newArgs << ( new ConstantExpression(section) )
+
+            // collect all variable tokens and pass them as single list argument
+            final variables = fetchVariables(closure,unit)
+            final listArg = new ArrayList(variables.size())
+            for( TokenValRef var: variables ) {
+                def pName = new ConstantExpression(var.name)
+                def pLine = new ConstantExpression(var.lineNum)
+                def pCol = new ConstantExpression(var.colNum)
+                listArg << createX( TokenValRef, pName, pLine, pCol )
+            }
+            newArgs << ( new ListExpression(listArg) )
+
+            // invokes the BodyDef constructor
+            createX( BodyDef, newArgs )
+        }
+
+        /**
+         * Read the user provided script source string
+         *
+         * @param node
+         * @param buffer
+         * @param unit
+         */
+        private void readSource( ASTNode node, StringBuilder buffer, SourceUnit unit, stripBrackets=false ) {
+            final colx = node.getColumnNumber()
+            final colz = node.getLastColumnNumber()
+            final first = node.getLineNumber()
+            final last = node.getLastLineNumber()
+            for( int i=first; i<=last; i++ ) {
+                def line = unit.source.getLine(i, null)
+                if( i==last ) {
+                    line = line.substring(0,colz-1)
+                    if( stripBrackets ) {
+                        line = line.replaceFirst(/}.*$/,'')
+                        if( !line.trim() ) continue
+                    }
+                }
+                if( i==first ) {
+                    line = line.substring(colx-1)
+                    if( stripBrackets ) {
+                        line = line.replaceFirst(/^.*\{/,'').trim()
+                        if( !line.trim() ) continue
+                    }
+                }
+                buffer.append(line) .append('\n')
+            }
+        }
+
+        protected void fixLazyGString( Statement stm ) {
+            if( stm instanceof ExpressionStatement && stm.getExpression() instanceof MethodCallExpression ) {
+                new GStringToLazyVisitor(unit).visitExpressionStatement(stm)
+            }
+        }
+
+        protected void fixDirectiveWithNegativeValue( Statement stm ) {
+            if( stm instanceof ExpressionStatement && stm.getExpression() instanceof BinaryExpression ) {
+                def binary = (BinaryExpression)stm.getExpression()
+                if(!(binary.leftExpression instanceof VariableExpression))
+                    return
+                if( binary.operation.type != Types.MINUS )
+                    return
+
+                // -- transform the binary expression into a method call expression
+                //    where the left expression represents the method name to invoke
+                def methodName = ((VariableExpression)binary.leftExpression).name
+
+                // -- wrap the value into a minus operator
+                def value = (Expression)new UnaryMinusExpression( binary.rightExpression )
+                def args = new ArgumentListExpression( [value] )
+
+                // -- create the method call expression and replace it to the binary expression
+                def call = new MethodCallExpression(new VariableExpression('this'), methodName, args)
+                stm.setExpression(call)
+
+            }
+        }
+
+        protected void fixStdinStdout( ExpressionStatement stm ) {
+
+            // transform the following syntax:
+            //      `stdin from x`  --> stdin() from (x)
+            //      `stdout into x` --> `stdout() into (x)`
+            VariableExpression varX
+            if( stm.expression instanceof PropertyExpression ) {
+                def expr = (PropertyExpression)stm.expression
+                def obj = expr.objectExpression
+                def prop = expr.property as ConstantExpression
+                def target = new VariableExpression(prop.text)
+
+                if( obj instanceof MethodCallExpression ) {
+                    def methodCall = obj as MethodCallExpression
+                    if( 'stdout' == methodCall.getMethodAsString() ) {
+                        def stdout = new MethodCallExpression( new VariableExpression('this'), 'stdout', new ArgumentListExpression()  )
+                        def into = new MethodCallExpression(stdout, 'into', new ArgumentListExpression(target))
+                        // remove replace the old one with the new one
+                        stm.setExpression( into )
+                    }
+                    else if( 'stdin' == methodCall.getMethodAsString() ) {
+                        def stdin = new MethodCallExpression( new VariableExpression('this'), 'stdin', new ArgumentListExpression()  )
+                        def from = new MethodCallExpression(stdin, 'from', new ArgumentListExpression(target))
+                        // remove replace the old one with the new one
+                        stm.setExpression( from )
+                    }
+                }
+            }
+            // transform the following syntax:
+            //      `stdout into (x,y,..)` --> `stdout() into (x,y,..)`
+            else if( stm.expression instanceof MethodCallExpression ) {
+                def methodCall = (MethodCallExpression)stm.expression
+                if( 'stdout' == methodCall.getMethodAsString() ) {
+                    def args = methodCall.getArguments()
+                    if( args instanceof ArgumentListExpression && args.getExpressions() && args.getExpression(0) instanceof MethodCallExpression ) {
+                        def methodCall2 = (MethodCallExpression)args.getExpression(0)
+                        def args2 = methodCall2.getArguments()
+                        if( args2 instanceof ArgumentListExpression && methodCall2.methodAsString == 'into') {
+                            def vars = args2.getExpressions()
+                            def stdout = new MethodCallExpression( new VariableExpression('this'), 'stdout', new ArgumentListExpression()  )
+                            def into = new MethodCallExpression(stdout, 'into', new ArgumentListExpression(vars))
+                            // remove replace the old one with the new one
+                            stm.setExpression( into )
+                        }
+                    }
+                }
+            }
+            else if( (varX=isVariableX(stm.expression)) && (varX.name=='stdin' || varX.name=='stdout') && NF.isDsl2() ) {
+                final name = varX.name=='stdin' ? '_in_stdin' : '_out_stdout'
+                final call = new MethodCallExpression( new VariableExpression('this'), name, new ArgumentListExpression()  )
+                // remove replace the old one with the new one
+                stm.setExpression(call)
+            }
+        }
+
+        /*
+         * handle *input* parameters
+         */
+        protected void convertInputMethod( Expression expression ) {
+            log.trace "convert > input expression: $expression"
+
+            if( expression instanceof MethodCallExpression ) {
+
+                def methodCall = expression as MethodCallExpression
+                def methodName = methodCall.getMethodAsString()
+                def nested = methodCall.objectExpression instanceof MethodCallExpression
+                log.trace "convert > input method: $methodName"
+
+                if( methodName in ['val','env','file','each','set','stdin','path','tuple'] ) {
+                    //this methods require a special prefix
+                    if( !nested )
+                        methodCall.setMethod( new ConstantExpression('_in_' + methodName) )
+
+                    fixMethodCall(methodCall)
+                }
+
+                /*
+                 * Handles a GString a file name, like this:
+                 *
+                 *      input:
+                 *        file x name "$var_name" from q
+                 *
+                 */
+                else if( methodName == 'name' && isWithinMethod(expression, 'file') ) {
+                    varToConstX(methodCall.getArguments())
+                }
+
+                // invoke on the next method call
+                if( expression.objectExpression instanceof MethodCallExpression ) {
+                    convertInputMethod(methodCall.objectExpression)
+                }
+            }
+
+            else if( expression instanceof PropertyExpression ) {
+                // invoke on the next method call
+                if( expression.objectExpression instanceof MethodCallExpression ) {
+                    convertInputMethod(expression.objectExpression)
+                }
+            }
+
+        }
+
+        protected boolean isWithinMethod(MethodCallExpression method, String name) {
+            if( method.objectExpression instanceof MethodCallExpression ) {
+                return isWithinMethod(method.objectExpression as MethodCallExpression, name)
+            }
+
+            return method.getMethodAsString() == name
+        }
+
+        /**
+         * Transform a map entry `emit: something` into `emit: 'something'
+         * (ie. as a constant) in a map expression passed as argument to
+         * a method call. This allow the syntax
+         *
+         *   output:
+         *   path 'foo', emit: bar
+         *
+         * @param call
+         */
+        protected void fixOutEmitOption(MethodCallExpression call) {
+            List args = isTupleX(call.arguments)?.expressions
+            if( !args ) return
+            if( args.size()<2 && (args.size()!=1 || call.methodAsString!='_out_stdout')) return
+            MapExpression map = isMapX(args[0])
+            if( !map ) return
+            for( int i=0; i output expression: $expression"
+
+            if( !(expression instanceof MethodCallExpression) ) {
+                return
+            }
+
+            def methodCall = expression as MethodCallExpression
+            def methodName = methodCall.getMethodAsString()
+            def nested = methodCall.objectExpression instanceof MethodCallExpression
+            log.trace "convert > output method: $methodName"
+
+            if( methodName in ['val','env','file','set','stdout','path','tuple'] && !nested ) {
+                // prefix the method name with the string '_out_'
+                methodCall.setMethod( new ConstantExpression('_out_' + methodName) )
+                fixMethodCall(methodCall)
+                fixOutEmitOption(methodCall)
+            }
+
+            else if( methodName in ['into','mode'] ) {
+                fixMethodCall(methodCall)
+            }
+
+            // continue to traverse
+            if( methodCall.objectExpression instanceof MethodCallExpression ) {
+                convertOutputMethod(methodCall.objectExpression)
+            }
+
+        }
+
+        private boolean withinTupleMethod
+
+        private boolean withinEachMethod
+
+        /**
+         * This method converts the a method call argument from a Variable to a Constant value
+         * so that it is possible to reference variable that not yet exist
+         *
+         * @param methodCall The method object for which it is required to change args definition
+         * @param flagVariable Whenever append a flag specified if the variable replacement has been applied
+         * @param index The index of the argument to modify
+         * @return
+         */
+        protected void fixMethodCall( MethodCallExpression methodCall ) {
+            final name = methodCall.methodAsString
+
+            withinTupleMethod = name == '_in_set' || name == '_out_set' || name == '_in_tuple' || name == '_out_tuple'
+            withinEachMethod = name == '_in_each'
+
+            try {
+                if( isOutputWithPropertyExpression(methodCall) ) {
+                    // transform an output value declaration such
+                    //   output: val( obj.foo )
+                    // to
+                    //   output: val({ obj.foo })
+                    wrapPropertyToClosure((ArgumentListExpression)methodCall.getArguments())
+                }
+                else
+                    varToConstX(methodCall.getArguments())
+
+            } finally {
+                withinTupleMethod = false
+                withinEachMethod = false
+            }
+        }
+
+        static final private List OUT_PROPERTY_VALID_TYPES = ['_out_val', '_out_env', '_out_file', '_out_path']
+
+        protected boolean isOutputWithPropertyExpression(MethodCallExpression methodCall) {
+            if( methodCall.methodAsString !in OUT_PROPERTY_VALID_TYPES  )
+                return false
+            if( methodCall.getArguments() instanceof ArgumentListExpression ) {
+                def args = (ArgumentListExpression)methodCall.getArguments()
+                if( args.size()==0 || args.size()>2 )
+                    return false
+
+                return args.last() instanceof PropertyExpression
+            }
+
+            return false
+        }
+
+        protected void wrapPropertyToClosure(ArgumentListExpression expr) {
+            final args = expr as ArgumentListExpression
+            final property = (PropertyExpression) args.last()
+            final closure = wrapPropertyToClosure(property)
+            args.getExpressions().set(args.size()-1, closure)
+        }
+
+        protected ClosureExpression wrapPropertyToClosure(PropertyExpression property)  {
+            def block = new BlockStatement()
+            block.addStatement( new ExpressionStatement(property) )
+
+            def closure = new ClosureExpression( Parameter.EMPTY_ARRAY, block )
+            closure.variableScope = new VariableScope(block.variableScope)
+
+            return closure
+        }
+
+
+        protected Expression varToStrX( Expression expr ) {
+            if( expr instanceof VariableExpression ) {
+                def name = ((VariableExpression) expr).getName()
+                return createX( LazyVar, new ConstantExpression(name) )
+            }
+            else if( expr instanceof PropertyExpression ) {
+                // transform an output declaration such
+                // output: tuple val( obj.foo )
+                //  to
+                // output: tuple val({ obj.foo })
+                return wrapPropertyToClosure(expr)
+            }
+
+            if( expr instanceof TupleExpression )  {
+                def i = 0
+                def list = expr.getExpressions()
+                for( Expression item : list ) {
+                    list[i++] = varToStrX(item)
+                }
+
+                return expr
+            }
+
+            return expr
+        }
+
+        protected Expression varToConstX( Expression expr ) {
+
+            if( expr instanceof VariableExpression ) {
+                // when it is a variable expression, replace it with a constant representing
+                // the variable name
+                def name = ((VariableExpression) expr).getName()
+
+                /*
+                 * the 'stdin' is used as placeholder for the standard input in the tuple definition. For example:
+                 *
+                 * input:
+                 *    tuple( stdin, .. ) from q
+                 */
+                if( name == 'stdin' && withinTupleMethod )
+                    return createX( TokenStdinCall )
+
+                /*
+                 * input:
+                 *    tuple( stdout, .. )
+                 */
+                else if ( name == 'stdout' && withinTupleMethod )
+                    return createX( TokenStdoutCall )
+
+                else
+                    return createX( LazyVar, new ConstantExpression(name) )
+            }
+
+            if( expr instanceof MethodCallExpression ) {
+                def methodCall = expr as MethodCallExpression
+
+                /*
+                 * replace 'file' method call in the tuple definition, for example:
+                 *
+                 * input:
+                 *   tuple( file(fasta:'*.fa'), .. ) from q
+                 */
+                if( methodCall.methodAsString == 'file' && (withinTupleMethod || withinEachMethod) ) {
+                    def args = (TupleExpression) varToConstX(methodCall.arguments)
+                    return createX( TokenFileCall, args )
+                }
+                else if( methodCall.methodAsString == 'path' && (withinTupleMethod || withinEachMethod) ) {
+                    def args = (TupleExpression) varToConstX(methodCall.arguments)
+                    return createX( TokenPathCall, args )
+                }
+
+                /*
+                 * input:
+                 *  tuple( env(VAR_NAME) ) from q
+                 */
+                if( methodCall.methodAsString == 'env' && withinTupleMethod ) {
+                    def args = (TupleExpression) varToStrX(methodCall.arguments)
+                    return createX( TokenEnvCall, args )
+                }
+
+                /*
+                 * input:
+                 *   tuple val(x), .. from q
+                 */
+                if( methodCall.methodAsString == 'val' && withinTupleMethod ) {
+                    def args = (TupleExpression) varToStrX(methodCall.arguments)
+                    return createX( TokenValCall, args )
+                }
+
+            }
+
+            // -- TupleExpression or ArgumentListExpression
+            if( expr instanceof TupleExpression )  {
+                def i = 0
+                def list = expr.getExpressions()
+                for( Expression item : list )  {
+                    list[i++] = varToConstX(item)
+                }
+                return expr
+            }
+
+            return expr
+        }
+
+        /**
+         * Wrap a generic expression with in a closure expression
+         *
+         * @param block The block to which the resulting closure has to be appended
+         * @param expr The expression to the wrapped in a closure
+         * @param len
+         * @return A tuple in which:
+         *      
  • 1st item: {@code true} if successful or {@code false} otherwise
+         *      
  • 2nd item: on error condition the line containing the error in the source script, zero otherwise
+         *      
  • 3rd item: on error condition the column containing the error in the source script, zero otherwise
+         *
+         */
+        protected boolean wrapExpressionWithClosure( BlockStatement block, Expression expr, int len, CharSequence source, SourceUnit unit ) {
+            if( expr instanceof GStringExpression || expr instanceof ConstantExpression ) {
+                // remove the last expression
+                block.statements.remove(len-1)
+
+                // and replace it by a wrapping closure
+                def closureExp = new ClosureExpression( Parameter.EMPTY_ARRAY, new ExpressionStatement(expr) )
+                closureExp.variableScope = new VariableScope(block.variableScope)
+
+                // append to the list of statement
+                //def wrap = newObj(BodyDef, closureExp, new ConstantExpression(source.toString()), ConstantExpression.TRUE)
+                def wrap = makeScriptWrapper(closureExp, source, 'script', unit )
+                block.statements.add( new ExpressionStatement(wrap) )
+
+                return true
+            }
+            else if( expr instanceof ClosureExpression ) {
+                // do not touch it
+                return true
+            }
+            else {
+                log.trace "Invalid process result expression: ${expr} -- Only constant or string expression can be used"
+            }
+
+            return false
+        }
+
+        protected boolean isIllegalName(String name, ASTNode node) {
+            if( name in RESERVED_NAMES ) {
+                unit.addError( new SyntaxException("Identifier `$name` is reserved for internal use", node.lineNumber, node.columnNumber+8) )
+                return true
+            }
+            if( name in workflowNames || name in processNames ) {
+                unit.addError( new SyntaxException("Identifier `$name` is already used by another definition", node.lineNumber, node.columnNumber+8) )
+                return true
+            }
+            if( name.contains(SCOPE_SEP) ) {
+                def offset =  8+2+ name.indexOf(SCOPE_SEP)
+                unit.addError( new SyntaxException("Process and workflow names cannot contain colon character", node.lineNumber, node.columnNumber+offset) )
+                return true
+            }
+            return false
+        }
+
+        /**
+         * This method handle the process definition, so that it transform the user entered syntax
+         *    process myName ( named: args, ..  ) { code .. }
+         *
+         * into
+         *    process ( [named:args,..], String myName )  { }
+         *
+         * @param methodCall
+         * @param unit
+         */
+        protected void convertProcessDef( MethodCallExpression methodCall, SourceUnit unit ) {
+            log.trace "Converts 'process' ${methodCall.arguments}"
+
+            assert methodCall.arguments instanceof ArgumentListExpression
+            def list = (methodCall.arguments as ArgumentListExpression).getExpressions()
+
+            // extract the first argument which has to be a method-call expression
+            // the name of this method represent the *process* name
+            if( list.size() != 1 || !list[0].class.isAssignableFrom(MethodCallExpression) ) {
+                log.debug "Missing name in process definition at line: ${methodCall.lineNumber}"
+                unit.addError( new SyntaxException("Process definition syntax error -- A string identifier must be provided after the `process` keyword", methodCall.lineNumber, methodCall.columnNumber+7))
+                return
+            }
+
+            def nested = list[0] as MethodCallExpression
+            def name = nested.getMethodAsString()
+            // check the process name is not defined yet
+            if( isIllegalName(name, methodCall) ) {
+                return
+            }
+            processNames.add(name)
+
+            // the nested method arguments are the arguments to be passed
+            // to the process definition, plus adding the process *name*
+            // as an extra item in the arguments list
+            def args = nested.getArguments() as ArgumentListExpression
+            log.trace "Process name: $name with args: $args"
+
+            // make sure to add the 'name' after the map item
+            // (which represent the named parameter attributes)
+            list = args.getExpressions()
+            if( list.size()>0 && list[0] instanceof MapExpression ) {
+                list.add(1, new ConstantExpression(name))
+            }
+            else {
+                list.add(0, new ConstantExpression(name))
+            }
+
+            // set the new list as the new arguments
+            methodCall.setArguments( args )
+
+            // now continue as before !
+            convertProcessBlock(methodCall, unit)
+        }
+
+        /**
+         * Fetch all the variable references in a closure expression.
+         *
+         * @param closure
+         * @param unit
+         * @return The set of variable names referenced in the script. NOTE: it includes properties in the form {@code object.propertyName}
+         */
+        protected Set fetchVariables( ClosureExpression closure, SourceUnit unit ) {
+            def visitor = new VariableVisitor(unit)
+            visitor.visitClosureExpression(closure)
+            return visitor.allVariables
+        }
+
+    }
+
+}
diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/NextflowXformImpl.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/NextflowXformImpl.groovy
index 4d3c4fa2a5..e4359118d3 100644
--- a/modules/nextflow/src/main/groovy/nextflow/ast/NextflowXformImpl.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/ast/NextflowXformImpl.groovy
@@ -32,7 +32,8 @@ import org.codehaus.groovy.control.SourceUnit
 import org.codehaus.groovy.transform.ASTTransformation
 import org.codehaus.groovy.transform.GroovyASTTransformation
 /**
- * Implements the syntax transformations for Nextflow scripts.
+ * Implements Nextflow Xform logic
+ * See http://groovy-lang.org/metaprogramming.html#_classcodeexpressiontransformer
  *
  * @author Paolo Di Tommaso 
  */
@@ -41,14 +42,91 @@ import org.codehaus.groovy.transform.GroovyASTTransformation
 @GroovyASTTransformation(phase = CompilePhase.CONVERSION)
 class NextflowXformImpl implements ASTTransformation {
 
+    SourceUnit unit
+
     @Override
-    void visit(ASTNode[] nodes, SourceUnit unit) {
-        final classNode = (ClassNode)nodes[1]
-        new BinaryExpressionXform(unit).visitClass(classNode)
-        new DslCodeVisitor(unit).visitClass(classNode)
-        new OperatorXform(unit).visitClass(classNode)
-        new ProcessFnXform(unit).visitClass(classNode)
-        new WorkflowFnXform(unit).visitClass(classNode)
+    void visit(ASTNode[] nodes, SourceUnit source) {
+        this.unit = unit
+        createVisitor().visitClass((ClassNode)nodes[1])
+    }
+
+    protected ClassCodeExpressionTransformer createVisitor() {
+
+        new ClassCodeExpressionTransformer() {
+
+            protected SourceUnit getSourceUnit() { unit }
+
+            @Override
+            Expression transform(Expression expr) {
+                if (expr == null)
+                    return null
+
+                def newExpr = transformBinaryExpression(expr)
+                if( newExpr ) {
+                    return newExpr
+                }
+                else if( expr instanceof ClosureExpression) {
+                    visitClosureExpression(expr)
+                }
+
+                return super.transform(expr)
+            }
+
+            /**
+             * This method replaces the `==` with the invocation of
+             * {@link LangHelpers#compareEqual(java.lang.Object, java.lang.Object)}
+             *
+             * This is required to allow the comparisons of `Path` objects
+             * which by default are not supported because it implements the Comparator interface
+             *
+             * See
+             *  {@link LangHelpers#compareEqual(java.lang.Object, java.lang.Object)}
+             *  https://stackoverflow.com/questions/28355773/in-groovy-why-does-the-behaviour-of-change-for-interfaces-extending-compar#comment45123447_28387391
+             *
+             */
+            protected Expression transformBinaryExpression(Expression expr) {
+
+                if( expr.class != BinaryExpression )
+                    return null
+
+                def binary = expr as BinaryExpression
+                def left = binary.getLeftExpression()
+                def right = binary.getRightExpression()
+
+                if( '=='.equals(binary.operation.text) )
+                    return call('compareEqual',left,right)
+
+                if( '!='.equals(binary.operation.text) )
+                    return new NotExpression(call('compareEqual',left,right))
+
+                if( '<'.equals(binary.operation.text) )
+                    return call('compareLessThan', left,right)
+
+                if( '<='.equals(binary.operation.text) )
+                    return call('compareLessThanEqual', left,right)
+
+                if( '>'.equals(binary.operation.text) )
+                    return call('compareGreaterThan', left,right)
+
+                if( '>='.equals(binary.operation.text) )
+                    return call('compareGreaterThanEqual', left,right)
+
+                return null
+            }
+
+
+            private MethodCallExpression call(String method, Expression left, Expression right) {
+
+                final a = transformBinaryExpression(left) ?: left
+                final b = transformBinaryExpression(right) ?: right
+
+                GeneralUtils.callX(
+                        GeneralUtils.classX(LangHelpers),
+                        method,
+                        GeneralUtils.args(a,b))
+            }
+
+        }
     }
 
 }
diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/WorkflowFn.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/OpXform.groovy
similarity index 77%
rename from modules/nextflow/src/main/groovy/nextflow/ast/WorkflowFn.groovy
rename to modules/nextflow/src/main/groovy/nextflow/ast/OpXform.groovy
index 77dfcce8e2..76c6d86ba6 100644
--- a/modules/nextflow/src/main/groovy/nextflow/ast/WorkflowFn.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/ast/OpXform.groovy
@@ -21,16 +21,12 @@ import java.lang.annotation.Retention
 import java.lang.annotation.RetentionPolicy
 import java.lang.annotation.Target
 
+import org.codehaus.groovy.transform.GroovyASTTransformationClass
+
 /**
- * Annotation for workflow functions.
- *
- * @author Ben Sherman 
+ * Declares Nextflow operators AST xforms
  */
-@Retention(RetentionPolicy.RUNTIME)
+@Retention(RetentionPolicy.SOURCE)
 @Target(ElementType.METHOD)
-@interface WorkflowFn {
-    boolean main() default false
-
-    // injected via AST transform
-    String source()
-}
+@GroovyASTTransformationClass(classes = [OpXformImpl])
+@interface OpXform {}
diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/OperatorXform.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/OpXformImpl.groovy
similarity index 90%
rename from modules/nextflow/src/main/groovy/nextflow/ast/OperatorXform.groovy
rename to modules/nextflow/src/main/groovy/nextflow/ast/OpXformImpl.groovy
index 8692fc01fc..c60a2a341e 100644
--- a/modules/nextflow/src/main/groovy/nextflow/ast/OperatorXform.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/ast/OpXformImpl.groovy
@@ -60,7 +60,7 @@ import static org.codehaus.groovy.ast.tools.GeneralUtils.returnS
 import static org.codehaus.groovy.ast.tools.GeneralUtils.stmt
 import static org.codehaus.groovy.ast.tools.GeneralUtils.varX
 /**
- * Implements the syntax transformations for Nextflow operators.
+ * Implements Nextflow operators xform logic
  *
  * See http://groovy-lang.org/metaprogramming.html#_classcodeexpressiontransformer
  *
@@ -68,72 +68,18 @@ import static org.codehaus.groovy.ast.tools.GeneralUtils.varX
  */
 @Slf4j
 @CompileStatic
-class OperatorXform extends ClassCodeExpressionTransformer {
+@GroovyASTTransformation(phase = CompilePhase.CONVERSION)
+class OpXformImpl implements ASTTransformation {
 
-    static final String BRANCH_METHOD_NAME = 'branch'
+    static final public String BRANCH_METHOD_NAME = 'branch'
 
-    static final String BRANCH_CRITERIA_FUN = 'branchCriteria'
+    static final public String BRANCH_CRITERIA_FUN = 'branchCriteria'
 
-    static final String MULTIMAP_METHOD_NAME = 'multiMap'
+    static final public String MULTIMAP_METHOD_NAME = 'multiMap'
 
-    static final String MULTIMAP_CRITERIA_FUN = 'multiMapCriteria'
+    static final public String MULTIMAP_CRITERIA_FUN = 'multiMapCriteria'
 
-    private final SourceUnit unit
-
-    OperatorXform(SourceUnit unit) {
-        this.unit = unit
-    }
-
-    @Override
-    protected SourceUnit getSourceUnit() { unit }
-
-    @Override
-    Expression transform(Expression expr) {
-        if (expr == null)
-            return null
-
-        ClosureExpression body
-        if( (body=isBranchOpCall(expr)) ) {
-            return new BranchTransformer(expr as MethodCallExpression, body).apply()
-        }
-        else if( (body=isMultiMapOpCall(expr)) ) {
-            return new MultiMapTransformer(expr as MethodCallExpression, body).apply()
-        }
-        else if( expr instanceof ClosureExpression) {
-            visitClosureExpression(expr)
-        }
-
-        return super.transform(expr)
-    }
-
-    protected ClosureExpression isBranchOpCall(Expression expr) {
-        final m = ASTHelpers.isMethodCallX(expr)
-        if( m ) {
-            final name = m.methodAsString
-            final args = isArgsX(m.arguments)
-            final ClosureExpression ret = args && args.size()>0 ? isClosureX(args.last()) : null
-            if( name==BRANCH_METHOD_NAME && args.size()==1 )
-                return ret
-            if( name==BRANCH_CRITERIA_FUN && args.size()==1 && m.objectExpression.text=='this')
-                return ret
-        }
-        return null
-    }
-
-    protected ClosureExpression isMultiMapOpCall(Expression expr) {
-        final m = ASTHelpers.isMethodCallX(expr)
-        if( m ) {
-            final name = m.methodAsString
-            final args = isArgsX(m.arguments)
-            final ClosureExpression ret = args && args.size()>0 ? isClosureX(args.last()) : null
-            if( name==MULTIMAP_METHOD_NAME && args.size()==1 )
-                return ret
-            if( name==MULTIMAP_CRITERIA_FUN && args.size()==1 && m.objectExpression.text=='this')
-                return ret
-
-        }
-        return null
-    }
+    SourceUnit unit
 
     static class BranchCondition {
         String label
@@ -146,6 +92,7 @@ class OperatorXform extends ClassCodeExpressionTransformer {
         }
     }
 
+    @CompileStatic
     class BranchTransformer {
 
         final List impl = new ArrayList<>(20)
@@ -404,6 +351,72 @@ class OperatorXform extends ClassCodeExpressionTransformer {
         }
     }
 
+    @Override
+    void visit(ASTNode[] nodes, SourceUnit source) {
+        this.unit = unit
+        createVisitor().visitClass((ClassNode)nodes[1])
+    }
+
+    protected ClosureExpression isBranchOpCall(Expression expr) {
+        final m = ASTHelpers.isMethodCallX(expr)
+        if( m ) {
+            final name = m.methodAsString
+            final args = isArgsX(m.arguments)
+            final ClosureExpression ret = args && args.size()>0 ? isClosureX(args.last()) : null
+            if( name==BRANCH_METHOD_NAME && args.size()==1 )
+                return ret
+            if( name==BRANCH_CRITERIA_FUN && args.size()==1 && m.objectExpression.text=='this')
+                return ret
+        }
+        return null
+    }
+
+    protected ClosureExpression isMultiMapOpCall(Expression expr) {
+        final m = ASTHelpers.isMethodCallX(expr)
+        if( m ) {
+            final name = m.methodAsString
+            final args = isArgsX(m.arguments)
+            final ClosureExpression ret = args && args.size()>0 ? isClosureX(args.last()) : null
+            if( name==MULTIMAP_METHOD_NAME && args.size()==1 )
+                return ret
+            if( name==MULTIMAP_CRITERIA_FUN && args.size()==1 && m.objectExpression.text=='this')
+                return ret
+
+        }
+        return null
+    }
+
+
+    /**
+     * Visit AST node to apply operator xforms
+     */
+    protected ClassCodeExpressionTransformer createVisitor() {
+
+        new ClassCodeExpressionTransformer() {
+
+            protected SourceUnit getSourceUnit() { unit }
+
+            @Override
+            Expression transform(Expression expr) {
+                if (expr == null)
+                    return null
+
+                ClosureExpression body
+                if( (body=isBranchOpCall(expr)) ) {
+                    return new BranchTransformer(expr as MethodCallExpression, body).apply()
+                }
+                else if( (body=isMultiMapOpCall(expr)) ) {
+                    return new MultiMapTransformer(expr as MethodCallExpression, body).apply()
+                }
+                else if( expr instanceof ClosureExpression) {
+                    visitClosureExpression(expr)
+                }
+
+                return super.transform(expr)
+            }
+        }
+    }
+
 }
 
 
diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/ProcessFnXform.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/ProcessFnXform.groovy
deleted file mode 100644
index 67db63a86c..0000000000
--- a/modules/nextflow/src/main/groovy/nextflow/ast/ProcessFnXform.groovy
+++ /dev/null
@@ -1,167 +0,0 @@
-/*
- * Copyright 2013-2023, Seqera Labs
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package nextflow.ast
-
-import static org.codehaus.groovy.ast.tools.GeneralUtils.*
-
-import groovy.transform.CompileStatic
-import groovy.util.logging.Slf4j
-import nextflow.processor.TaskConfig
-import org.codehaus.groovy.ast.ASTNode
-import org.codehaus.groovy.ast.AnnotationNode
-import org.codehaus.groovy.ast.ClassCodeVisitorSupport
-import org.codehaus.groovy.ast.ClassNode
-import org.codehaus.groovy.ast.MethodNode
-import org.codehaus.groovy.ast.Parameter
-import org.codehaus.groovy.ast.expr.ArgumentListExpression
-import org.codehaus.groovy.ast.expr.BinaryExpression
-import org.codehaus.groovy.ast.expr.ClosureExpression
-import org.codehaus.groovy.ast.expr.Expression
-import org.codehaus.groovy.ast.expr.ListExpression
-import org.codehaus.groovy.ast.expr.MethodCallExpression
-import org.codehaus.groovy.ast.expr.UnaryMinusExpression
-import org.codehaus.groovy.ast.expr.VariableExpression
-import org.codehaus.groovy.ast.stmt.BlockStatement
-import org.codehaus.groovy.ast.stmt.ExpressionStatement
-import org.codehaus.groovy.ast.stmt.Statement
-import org.codehaus.groovy.control.SourceUnit
-import org.codehaus.groovy.syntax.SyntaxException
-import org.codehaus.groovy.syntax.Types
-/**
- * Implements syntax transformations for process functions.
- *
- * @author Ben Sherman 
- */
-@Slf4j
-@CompileStatic
-class ProcessFnXform extends ClassCodeVisitorSupport {
-
-    private final SourceUnit unit
-
-    ProcessFnXform(SourceUnit unit) {
-        this.unit = unit
-    }
-
-    @Override
-    protected SourceUnit getSourceUnit() { unit }
-
-    @Override
-    void visitMethod(MethodNode method) {
-        final annotation = method.getAnnotations()
-                .find(a -> a.getClassNode().getName() == 'ProcessFn')
-
-        if( annotation )
-            transform(method, annotation)
-    }
-
-    /**
-     * Transform a ProcessFn annotation to be semantically valid.
-     *
-     * @param method
-     * @param annotation
-     */
-    protected void transform(MethodNode method, AnnotationNode annotation) {
-        // fix directives
-        final directives = annotation.getMember('directives')
-        if( directives != null && directives instanceof ClosureExpression ) {
-            final block = (BlockStatement)directives.getCode()
-            for( Statement stmt : block.getStatements() )
-                fixDirectiveWithNegativeValue(stmt)
-        }
-
-        // TODO: append stub source
-
-        // append method params
-        final params = method.getParameters() as List
-        annotation.addMember( 'params', new ListExpression(
-            params.collect(p -> (Expression)constX(p.getName()))
-        ) )
-
-        // append script source
-        annotation.addMember( 'source', constX( getSource(method.getCode()) ) )
-
-        // append variable references so that global vars can be cached
-        final vars = getVariableRefs(method.getCode())
-        annotation.addMember( 'vars', new ListExpression(
-            vars.collect(var -> (Expression)constX(var))
-        ) )
-
-        // prepend `task` method parameter
-        params.push(new Parameter(new ClassNode(TaskConfig), 'task'))
-        method.setParameters(params as Parameter[])
-    }
-
-    /**
-     * Fix directives with a single argument with a negative value,
-     * since it will be parsed as a subtract expression if there are
-     * no parentheses.
-     *
-     * @param stmt
-     */
-    protected void fixDirectiveWithNegativeValue(Statement stmt) {
-        // -- check for binary subtract expression
-        if( stmt !instanceof ExpressionStatement )
-            return
-        def expr = ((ExpressionStatement)stmt).getExpression()
-        if( expr !instanceof BinaryExpression )
-            return
-        def binary = (BinaryExpression)expr
-        if( binary.leftExpression !instanceof VariableExpression )
-            return
-        if( binary.operation.type != Types.MINUS )
-            return
-
-        // -- transform binary expression `NAME - ARG` into method call `NAME(-ARG)`
-        def name = ((VariableExpression)binary.leftExpression).name
-        def arg = (Expression)new UnaryMinusExpression(binary.rightExpression)
-
-        ((ExpressionStatement)stmt).setExpression( new MethodCallExpression(
-            VariableExpression.THIS_EXPRESSION,
-            name,
-            new ArgumentListExpression(arg)
-        ) )
-    }
-
-    protected List getVariableRefs(Statement stmt) {
-        final visitor = new VariableVisitor(unit)
-        stmt.visit(visitor)
-        return visitor.getAllVariables().collect( ref -> ref.name )
-    }
-
-    private String getSource(ASTNode node) {
-        final buffer = new StringBuilder()
-        final colx = node.getColumnNumber()
-        final colz = node.getLastColumnNumber()
-        final first = node.getLineNumber()
-        final last = node.getLastLineNumber()
-        for( int i=first; i<=last; i++ ) {
-            def line = unit.source.getLine(i, null)
-            if( i==last )
-                line = line.substring(0,colz-1)
-            if( i==first )
-                line = line.substring(colx-1)
-            buffer.append(line) .append('\n')
-        }
-
-        return buffer.toString()
-    }
-
-    protected void syntaxError(ASTNode node, String message) {
-        unit.addError( new SyntaxException(message, node.lineNumber, node.columnNumber) )
-    }
-
-}
diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/WorkflowFnXform.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/WorkflowFnXform.groovy
deleted file mode 100644
index 94406d5885..0000000000
--- a/modules/nextflow/src/main/groovy/nextflow/ast/WorkflowFnXform.groovy
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Copyright 2013-2023, Seqera Labs
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package nextflow.ast
-
-import static org.codehaus.groovy.ast.tools.GeneralUtils.*
-
-import groovy.transform.CompileStatic
-import groovy.util.logging.Slf4j
-import org.codehaus.groovy.ast.ASTNode
-import org.codehaus.groovy.ast.AnnotationNode
-import org.codehaus.groovy.ast.ClassCodeVisitorSupport
-import org.codehaus.groovy.ast.MethodNode
-import org.codehaus.groovy.control.SourceUnit
-/**
- * Implements syntax transformations for workflow functions.
- *
- * @author Ben Sherman 
- */
-@Slf4j
-@CompileStatic
-class WorkflowFnXform extends ClassCodeVisitorSupport {
-
-    private final SourceUnit unit
-
-    WorkflowFnXform(SourceUnit unit) {
-        this.unit = unit
-    }
-
-    @Override
-    protected SourceUnit getSourceUnit() { unit }
-
-    @Override
-    void visitMethod(MethodNode method) {
-        final annotation = method.getAnnotations()
-                .find(a -> a.getClassNode().getName() == 'WorkflowFn')
-
-        if( annotation )
-            transform(method, annotation)
-    }
-
-    protected void transform(MethodNode method, AnnotationNode annotation) {
-        // append workflow source
-        annotation.addMember( 'source', constX( getSource(method.getCode()) ) )
-    }
-
-    private String getSource(ASTNode node) {
-        final buffer = new StringBuilder()
-        final colx = node.getColumnNumber()
-        final colz = node.getLastColumnNumber()
-        final first = node.getLineNumber()
-        final last = node.getLastLineNumber()
-        for( int i=first; i<=last; i++ ) {
-            def line = unit.source.getLine(i, null)
-            if( i==last )
-                line = line.substring(0,colz-1)
-            if( i==first )
-                line = line.substring(colx-1)
-            buffer.append(line) .append('\n')
-        }
-
-        return buffer.toString()
-    }
-
-}
diff --git a/modules/nextflow/src/main/groovy/nextflow/config/ConfigParser.groovy b/modules/nextflow/src/main/groovy/nextflow/config/ConfigParser.groovy
index 4145727256..af05ba8687 100644
--- a/modules/nextflow/src/main/groovy/nextflow/config/ConfigParser.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/config/ConfigParser.groovy
@@ -21,6 +21,7 @@ import java.nio.file.Path
 import ch.artecat.grengine.Grengine
 import com.google.common.hash.Hashing
 import groovy.transform.PackageScope
+import nextflow.ast.NextflowXform
 import nextflow.exception.ConfigParseException
 import nextflow.extension.Bolts
 import nextflow.file.FileHelper
@@ -171,6 +172,7 @@ class ConfigParser {
         if( renderClosureAsString )
             params.put('renderClosureAsString', true)
         config.addCompilationCustomizers(new ASTTransformationCustomizer(params, ConfigTransform))
+        config.addCompilationCustomizers(new ASTTransformationCustomizer(NextflowXform))
         //  add implicit types
         def importCustomizer = new ImportCustomizer()
         importCustomizer.addImports( Duration.name )
diff --git a/modules/nextflow/src/main/groovy/nextflow/config/ConfigTransformImpl.groovy b/modules/nextflow/src/main/groovy/nextflow/config/ConfigTransformImpl.groovy
index c65ea5af20..08c928115b 100644
--- a/modules/nextflow/src/main/groovy/nextflow/config/ConfigTransformImpl.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/config/ConfigTransformImpl.groovy
@@ -18,7 +18,6 @@ package nextflow.config
 
 import groovy.transform.CompileStatic
 import groovy.util.logging.Slf4j
-import nextflow.ast.BinaryExpressionXform
 import org.codehaus.groovy.ast.ASTNode
 import org.codehaus.groovy.ast.AnnotationNode
 import org.codehaus.groovy.ast.ClassCodeVisitorSupport
@@ -56,7 +55,6 @@ class ConfigTransformImpl implements ASTTransformation {
         // the following line is mostly an hack to pass a parameter to this xform instance
         this.renderClosureAsString = annot.getMember('renderClosureAsString') != null
         createVisitor(unit).visitClass(clazz)
-        new BinaryExpressionXform(unit).visitClass(clazz)
     }
 
     protected ClassCodeVisitorSupport createVisitor(SourceUnit unit) {
diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskConfig.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskConfig.groovy
index 05fd32a52f..564f0cd3f7 100644
--- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskConfig.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskConfig.groovy
@@ -23,7 +23,7 @@ import java.nio.file.Path
 import groovy.transform.CompileStatic
 import groovy.transform.PackageScope
 import nextflow.Const
-import nextflow.ast.DslCodeVisitor
+import nextflow.ast.NextflowDSLImpl
 import nextflow.exception.AbortOperationException
 import nextflow.exception.FailedGuardException
 import nextflow.executor.BashWrapperBuilder
@@ -510,7 +510,7 @@ class TaskConfig extends LazyMap implements Cloneable {
 
 
     protected TaskClosure getStubBlock() {
-        final code = target.get(DslCodeVisitor.PROCESS_STUB)
+        final code = target.get(NextflowDSLImpl.PROCESS_STUB)
         if( !code )
             return null
         if( code instanceof TaskClosure )
diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy
index 0e504b4286..2a39ae4798 100644
--- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy
@@ -50,7 +50,7 @@ import groovyx.gpars.group.PGroup
 import nextflow.NF
 import nextflow.Nextflow
 import nextflow.Session
-import nextflow.ast.DslCodeVisitor
+import nextflow.ast.NextflowDSLImpl
 import nextflow.ast.TaskCmdXform
 import nextflow.ast.TaskTemplateVarsXform
 import nextflow.cloud.CloudSpotTerminationException
@@ -466,11 +466,8 @@ class TaskProcessor {
             task.resolve(block)
         }
         else {
-            // -- prepend task config to arguments (for process function)
-            values.push(task.config)
-
             // -- resolve the task command script
-            task.resolve(taskBody, values.toArray())
+            task.resolve(taskBody)
         }
 
         // -- verify if exists a stored result for this case,
@@ -1721,7 +1718,7 @@ class TaskProcessor {
     protected boolean checkWhenGuard(TaskRun task) {
 
         try {
-            def pass = task.config.getGuard(DslCodeVisitor.PROCESS_WHEN)
+            def pass = task.config.getGuard(NextflowDSLImpl.PROCESS_WHEN)
             if( pass ) {
                 return true
             }
diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy
index a586b31f26..880ad24ea7 100644
--- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy
@@ -41,7 +41,6 @@ import nextflow.script.TaskClosure
 import nextflow.script.bundle.ResourcesBundle
 import nextflow.spack.SpackCache
 import nextflow.util.ArrayBag
-import org.codehaus.groovy.runtime.MethodClosure
 /**
  * Models a task instance
  *
@@ -620,24 +619,14 @@ class TaskRun implements Cloneable {
      * 2) extract the process code `source`
      * 3) assign the `script` code to execute
      *
-     * @param body
-     * @param args
+     * @param body A {@code BodyDef} object instance
      */
-    @PackageScope void resolve(BodyDef body, Object[] args) {
+    @PackageScope void resolve(BodyDef body) {
 
         // -- initialize the task code to be executed
-        this.code = body.closure
-
-        if( code instanceof MethodClosure ) {
-            // -- invoke task closure with arguments
-            code = code.curry(args)
-        }
-        else {
-            // -- invoke task closure with delegate
-            code = code.clone() as Closure
-            code.setDelegate(this.context)
-            code.setResolveStrategy(Closure.DELEGATE_ONLY)
-        }
+        this.code = body.closure.clone() as Closure
+        this.code.delegate = this.context
+        this.code.setResolveStrategy(Closure.DELEGATE_ONLY)
 
         // -- set the task source
         this.body = body
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy b/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy
index 6bf7fde7c5..14e7e9708c 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy
@@ -24,13 +24,9 @@ import groovy.transform.CompileStatic
 import groovy.util.logging.Slf4j
 import nextflow.NextflowMeta
 import nextflow.Session
-import nextflow.ast.ProcessFn
-import nextflow.ast.WorkflowFn
 import nextflow.exception.AbortOperationException
 import nextflow.script.dsl.ProcessBuilder
 import nextflow.script.dsl.ProcessDsl
-import nextflow.script.dsl.ProcessInputsBuilder
-import nextflow.script.dsl.ProcessOutputsBuilder
 import nextflow.script.dsl.WorkflowBuilder
 /**
  * Any user defined script will extends this class, it provides the base execution context
@@ -151,18 +147,6 @@ abstract class BaseScript extends Script implements ExecutionContext {
         include .setSession(session)
     }
 
-    @Override
-    Object getProperty(String name) {
-        try {
-            ExecutionStack.binding().getProperty(name)
-        }
-        catch( MissingPropertyException e ) {
-            if( !ExecutionStack.withinWorkflow() )
-                throw e
-            binding.getProperty(name)
-        }
-    }
-
     /**
      * Invokes custom methods in the task execution context
      *
@@ -175,99 +159,10 @@ abstract class BaseScript extends Script implements ExecutionContext {
      */
     @Override
     Object invokeMethod(String name, Object args) {
-        try {
-            ExecutionStack.binding().invokeMethod(name, args)
-        }
-        catch( MissingMethodException e ) {
-            if( !ExecutionStack.withinWorkflow() )
-                throw e
-            binding.invokeMethod(name, args)
-        }
-    }
-
-    private void applyDsl(Object delegate, Class clazz) {
-        final cl = clazz.newInstance(this, this)
-        cl.delegate = delegate
-        cl.resolveStrategy = Closure.DELEGATE_FIRST
-        cl.call()
-    }
-
-    private void registerProcessFn(Method method) {
-        final name = method.getName()
-        final processFn = method.getAnnotation(ProcessFn)
-
-        // validate annotation
-        if( processFn.script() && processFn.shell() )
-            throw new IllegalArgumentException("Process function `${name}` cannot have script and shell enabled simultaneously")
-
-        // build process from annotation
-        final builder = new ProcessBuilder(this, name)
-
-        // -- directives
-        applyDsl(builder, processFn.directives())
-
-        // -- inputs
-        final inputs = new ProcessInputsBuilder()
-        applyDsl(inputs, processFn.inputs())
-
-        for( String param : processFn.params() )
-            inputs.take(param)
-
-        // -- outputs
-        final outputs = new ProcessOutputsBuilder()
-        applyDsl(outputs, processFn.outputs())
-
-        // -- process type
-        final type =
-            processFn.script() ? 'script'
-            : processFn.shell() ? 'shell'
-            : 'exec'
-
-        // -- variable references
-        final valRefs = processFn.vars().collect( var -> new TokenValRef(var) )
-
-        // -- build process
-        final process = builder
-            .withInputs(inputs.build())
-            .withOutputs(outputs.build())
-            .withBody(this.&"${name}", type, processFn.source(), valRefs)
-            .build()
-
-        // register process
-        meta.addDefinition(process)
-    }
-
-    private void registerWorkflowFn(Method method) {
-        final name = method.getName()
-        final workflowFn = method.getAnnotation(WorkflowFn)
-
-        // build workflow from annotation
-        final builder = workflowFn.main()
-            ? new WorkflowBuilder(this)
-            : new WorkflowBuilder(this, name)
-
-        // create body
-        final body = new BodyDef( this.&"${name}", workflowFn.source(), 'workflow', [] )
-        builder.withBody(body)
-
-        // register workflow
-        final workflow = builder.build()
-        if( workflowFn.main() )
-            this.entryFlow = workflow
-        meta.addDefinition(workflow)
+        binding.invokeMethod(name, args)
     }
 
     private run0() {
-        // register any process and workflow functions
-        final clazz = this.getClass()
-        for( final method : clazz.getDeclaredMethods() ) {
-            if( method.isAnnotationPresent(ProcessFn) )
-                registerProcessFn(method)
-            if( method.isAnnotationPresent(WorkflowFn) )
-                registerWorkflowFn(method)
-        }
-
-        // execute script
         final result = runScript()
         if( meta.isModule() ) {
             return result
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ChannelOut.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ChannelOut.groovy
index 0187d59f9e..66a0c99f42 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ChannelOut.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ChannelOut.groovy
@@ -19,8 +19,8 @@ package nextflow.script
 import groovy.transform.CompileStatic
 import groovy.util.logging.Slf4j
 import groovyx.gpars.dataflow.DataflowWriteChannel
-import nextflow.ast.DslCodeVisitor
 import nextflow.exception.DuplicateChannelNameException
+import static nextflow.ast.NextflowDSLImpl.OUT_PREFIX
 /**
  * Models the output of a process or a workflow component returning
  * more than one output channels
@@ -65,9 +65,7 @@ class ChannelOut implements List {
         target = Collections.unmodifiableList(onlyWithName)
     }
 
-    Set getNames() {
-        channels.keySet().findAll { !it.startsWith(DslCodeVisitor.OUT_PREFIX) }
-    }
+    Set getNames() { channels.keySet().findAll { !it.startsWith(OUT_PREFIX) }  }
 
     @Override
     def getProperty(String name) {
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessConfig.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessConfig.groovy
index b85a135856..65fcc1efff 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessConfig.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessConfig.groovy
@@ -134,10 +134,10 @@ class ProcessConfig implements Map, Cloneable {
                 return isCacheable()
 
             case 'ext':
-                if( !configProperties.containsKey(name) ) {
-                    configProperties.put(name, new HashMap())
+                if( !configProperties.containsKey('ext') ) {
+                    configProperties.put('ext', new HashMap())
                 }
-                return configProperties.get(name)
+                return configProperties.get('ext')
 
             default:
                 if( configProperties.containsKey(name) )
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy
index 0cad157f6b..035588695f 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy
@@ -141,35 +141,6 @@ class ProcessDef extends BindableDef implements IterableDef, ChainableDef {
         return "Process `$name` declares ${expected} input ${ch} but ${actual} were specified"
     }
 
-    @Override
-    Object run(Object[] args) {
-        // initialise process config
-        initialize()
-
-        // create input channel
-        final source = collectInputs(args)
-
-        // set output channels
-        final singleton = !CH.isChannelQueue(source)
-        collectOutputs(singleton)
-
-        // create the executor
-        final executor = session
-                .executorFactory
-                .getExecutor(processName, config, taskBody, session)
-
-        // create processor class
-        session
-                .newProcessFactory(owner)
-                .newTaskProcessor(processName, executor, config, taskBody)
-                .run(source)
-
-        // the result channels
-        // note: the result object must be an array instead of a List to allow process
-        // composition ie. to use the process output as the input in another process invocation
-        return output = new ChannelOut(declaredOutputs)
-    }
-
     private DataflowReadChannel collectInputs(Object[] args0) {
         final args = ChannelOut.spread(args0)
         if( args.size() != declaredInputs.size() )
@@ -219,4 +190,33 @@ class ProcessDef extends BindableDef implements IterableDef, ChainableDef {
         }
     }
 
+    @Override
+    Object run(Object[] args) {
+        // initialise process config
+        initialize()
+
+        // create input channel
+        final source = collectInputs(args)
+
+        // set output channels
+        // note: the result object must be an array instead of a List to allow process
+        // composition ie. to use the process output as the input in another process invocation
+        final singleton = !CH.isChannelQueue(source)
+        collectOutputs(singleton)
+
+        // create the executor
+        final executor = session
+                .executorFactory
+                .getExecutor(processName, config, taskBody, session)
+
+        // create processor class
+        session
+                .newProcessFactory(owner)
+                .newTaskProcessor(processName, executor, config, taskBody)
+                .run(source)
+
+        // the result channels
+        return output = new ChannelOut(declaredOutputs)
+    }
+
 }
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ScriptMeta.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ScriptMeta.groovy
index 3b9e5dd622..784b40d784 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ScriptMeta.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ScriptMeta.groovy
@@ -25,8 +25,6 @@ import groovy.transform.Memoized
 import groovy.transform.PackageScope
 import groovy.util.logging.Slf4j
 import nextflow.NF
-import nextflow.ast.ProcessFn
-import nextflow.ast.WorkflowFn
 import nextflow.exception.DuplicateModuleFunctionException
 import nextflow.exception.MissingModuleComponentException
 import nextflow.script.bundle.ResourcesBundle
@@ -177,8 +175,6 @@ class ScriptMeta {
             if( Modifier.isStatic(method.getModifiers())) continue
             if( method.name.startsWith('super$')) continue
             if( method.name in INVALID_FUNCTION_NAMES ) continue
-            if( method.isAnnotationPresent(ProcessFn) ) continue
-            if( method.isAnnotationPresent(WorkflowFn) ) continue
 
             // If method is already into the list, maybe with other signature, it's not necessary to include it again
             if( result.find{it.name == method.name}) continue
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ScriptParser.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ScriptParser.groovy
index 05eb862b3b..e491124003 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ScriptParser.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ScriptParser.groovy
@@ -23,9 +23,9 @@ import groovy.transform.CompileStatic
 import nextflow.Channel
 import nextflow.Nextflow
 import nextflow.Session
+import nextflow.ast.NextflowDSL
 import nextflow.ast.NextflowXform
-import nextflow.ast.ProcessFn
-import nextflow.ast.WorkflowFn
+import nextflow.ast.OpXform
 import nextflow.exception.ScriptCompilationException
 import nextflow.extension.FilesEx
 import nextflow.file.FileHelper
@@ -114,16 +114,16 @@ class ScriptParser {
         importCustomizer.addImports( Channel.name )
         importCustomizer.addImports( Duration.name )
         importCustomizer.addImports( MemoryUnit.name )
-        importCustomizer.addImports( ProcessFn.name )
         importCustomizer.addImports( ValueObject.name )
-        importCustomizer.addImports( WorkflowFn.name )
         importCustomizer.addImport( 'channel', Channel.name )
         importCustomizer.addStaticStars( Nextflow.name )
 
         config = new CompilerConfiguration()
         config.addCompilationCustomizers( importCustomizer )
         config.scriptBaseClass = BaseScript.class.name
+        config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowDSL))
         config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowXform))
+        config.addCompilationCustomizers( new ASTTransformationCustomizer(OpXform))
 
         if( session?.debug )
             config.debug = true
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ScriptTokens.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ScriptTokens.groovy
index 63cbecab3d..7f719ad1fd 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ScriptTokens.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ScriptTokens.groovy
@@ -64,7 +64,7 @@ class TokenPathCall {
  *   tuple( stdin, .. ) from x
  * 
  *
- * @see nextflow.ast.DslCodeVisitor
+ * @see nextflow.ast.NextflowDSLImpl
  * @see nextflow.script.dsl.ProcessDsl#_in_tuple(java.lang.Object[])
  */
 class TokenStdinCall { }
@@ -76,7 +76,7 @@ class TokenStdinCall { }
  *   tuple( stdout, .. ) into x
  * 
  *
- * @see nextflow.ast.DslCodeVisitor
+ * @see nextflow.ast.NextflowDSLImpl
  * @see nextflow.script.dsl.ProcessDsl#_out_tuple(java.util.Map,java.lang.Object[])
  */
 class TokenStdoutCall { }
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/WorkflowDef.groovy b/modules/nextflow/src/main/groovy/nextflow/script/WorkflowDef.groovy
index de7cde7351..3b5d4b5cf8 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/WorkflowDef.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/WorkflowDef.groovy
@@ -24,7 +24,6 @@ import nextflow.exception.MissingProcessException
 import nextflow.exception.MissingValueException
 import nextflow.exception.ScriptRuntimeException
 import nextflow.extension.CH
-import org.codehaus.groovy.runtime.MethodClosure
 /**
  * Models a script workflow component
  *
@@ -191,52 +190,15 @@ class WorkflowDef extends BindableDef implements ChainableDef, IterableDef, Exec
     }
 
     private Object run0(Object[] args) {
+        collectInputs(binding, args)
+        // invoke the workflow execution
         final closure = body.closure
-        if( closure instanceof MethodClosure ) {
-            // invoke the workflow function with args
-            final target = closure.owner
-            final name = closure.method
-            final meta = target.metaClass.getMetaMethod(name, args)
-            if( meta == null )
-                throw new MissingMethodException(name, target.getClass(), args)
-            final method = target.getClass().getMethod(name, meta.getNativeParameterTypes())
-            if( method == null )
-                throw new MissingMethodException(name, target.getClass(), args)
-            final result = method.invoke(target, args)
-
-            // apply return value to declared outputs, binding
-            normalizeOutput(result)
-        }
-        else {
-            // invoke the workflow closure with delegate
-            collectInputs(binding, args)
-            closure.setDelegate(binding)
-            closure.setResolveStrategy(Closure.DELEGATE_FIRST)
-            closure.call()
-        }
-
+        closure.delegate = binding
+        closure.setResolveStrategy(Closure.DELEGATE_FIRST)
+        closure.call()
         // collect the workflow outputs
         output = collectOutputs(declaredOutputs)
         return output
     }
 
-    private void normalizeOutput(Object result) {
-        if( CH.isChannel(result) )
-            result = ['$out0': result]
-
-        if( result instanceof List )
-            result = result.inject([:], (acc, value) -> { acc.put("\$out${acc.size()}".toString(), value); acc })
-
-        if( result instanceof Map ) {
-            for( def entry : result ) {
-                declaredOutputs.add(entry.key)
-                binding.setVariable(entry.key, entry.value)
-            }
-        }
-        else if( result instanceof ChannelOut )
-            log.debug "Workflow `$name` > ignoring multi-channel return value"
-        else if( result != null )
-            throw new ScriptRuntimeException("Workflow `$name` emitted unexpected value of type ${result.class.name} -- ${result}")
-    }
-
 }
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessBuilder.groovy b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessBuilder.groovy
index 6266892d0b..a85ff72243 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessBuilder.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessBuilder.groovy
@@ -19,7 +19,7 @@ package nextflow.script.dsl
 import java.util.regex.Pattern
 
 import groovy.util.logging.Slf4j
-import nextflow.ast.DslCodeVisitor
+import nextflow.ast.NextflowDSLImpl
 import nextflow.exception.IllegalConfigException
 import nextflow.exception.IllegalDirectiveException
 import nextflow.exception.ScriptRuntimeException
@@ -130,9 +130,9 @@ class ProcessBuilder {
     private void checkName(String name) {
         if( DIRECTIVES.contains(name) )
             return
-        if( name == DslCodeVisitor.PROCESS_WHEN )
+        if( name == NextflowDSLImpl.PROCESS_WHEN )
             return
-        if( name == DslCodeVisitor.PROCESS_STUB )
+        if( name == NextflowDSLImpl.PROCESS_STUB )
             return
 
         String message = "Unknown process directive: `$name`"
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessInputsBuilder.groovy b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessInputsBuilder.groovy
deleted file mode 100644
index ea37a702e1..0000000000
--- a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessInputsBuilder.groovy
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Copyright 2013-2023, Seqera Labs
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package nextflow.script.dsl
-
-import groovy.transform.CompileStatic
-import nextflow.script.ProcessFileInput
-import nextflow.script.ProcessInput
-import nextflow.script.ProcessInputs
-
-/**
- * Builder for {@link ProcessInputs}.
- *
- * @author Ben Sherman 
- */
-@CompileStatic
-class ProcessInputsBuilder {
-
-    private ProcessInputs inputs = new ProcessInputs()
-
-    ProcessInputsBuilder env(String name, Object source) {
-        inputs.addEnv(name, source)
-        return this
-    }
-
-    ProcessInputsBuilder path(Map opts=[:], Object source) {
-        inputs.addFile(new ProcessFileInput(source, null, true, opts))
-        return this
-    }
-
-    ProcessInputsBuilder stdin(Object source) {
-        inputs.stdin = source
-        return this
-    }
-
-    /**
-     * Declare a process input parameter which will be
-     * bound when the task is created and can be referenced by
-     * other process input methods. For example:
-     *
-     *   take 'sample_id'
-     *   take 'files'
-     *
-     *   env('SAMPLE_ID') { sample_id }
-     *   path { files }
-     *
-     * @param name
-     */
-    ProcessInputsBuilder take(String name) {
-        inputs.add(new ProcessInput(name))
-        return this
-    }
-
-    ProcessInputs build() {
-        return inputs
-    }
-
-}
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessOutputsBuilder.groovy b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessOutputsBuilder.groovy
deleted file mode 100644
index ada10c6bbf..0000000000
--- a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessOutputsBuilder.groovy
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Copyright 2013-2023, Seqera Labs
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package nextflow.script.dsl
-
-import groovy.transform.CompileStatic
-import nextflow.script.ProcessFileOutput
-import nextflow.script.ProcessOutput
-import nextflow.script.ProcessOutputs
-
-/**
- * Builder for {@link ProcessOutputs}.
- *
- * @author Ben Sherman 
- */
-@CompileStatic
-class ProcessOutputsBuilder {
-
-    private ProcessOutputs outputs = new ProcessOutputs()
-
-    ProcessOutputsBuilder env(String name) {
-        env(name, name)
-    }
-
-    ProcessOutputsBuilder env(String key, String target) {
-        outputs.addEnv(key, target)
-        return this
-    }
-
-    ProcessOutputsBuilder path(Map opts=[:], String name) {
-        path(opts, name, name)
-    }
-
-    ProcessOutputsBuilder path(Map opts=[:], String key, Object target) {
-        outputs.addFile(key, new ProcessFileOutput(target, true, opts))
-        return this
-    }
-
-    /**
-     * Declare a process output with a closure that will
-     * be evaluated after the task execution. For example:
-     *
-     *   env 'SAMPLE_ID'           // declare output env 'SAMPLE_ID'
-     *   path '$file0', 'file.txt' // declare output file 'file.txt'
-     *
-     *   emit { sample_id }        // variable 'sample_id' in task context
-     *   emit { stdout }           // standard output of task script
-     *   emit { [env('SAMPLE_ID'), path('$file0')] }
-     *   emit { new Sample(sample_id, path('$file0')) }
-     *
-     * @param opts
-     * @param target
-     */
-    ProcessOutputsBuilder emit(Map opts=[:], Object target) {
-        outputs.addParam(target, opts)
-        return this
-    }
-
-    ProcessOutputs build() {
-        outputs
-    }
-
-}
diff --git a/modules/nextflow/src/test/groovy/nextflow/ast/DslCodeVisitorTest.groovy b/modules/nextflow/src/test/groovy/nextflow/ast/NextflowDSLImplTest.groovy
similarity index 98%
rename from modules/nextflow/src/test/groovy/nextflow/ast/DslCodeVisitorTest.groovy
rename to modules/nextflow/src/test/groovy/nextflow/ast/NextflowDSLImplTest.groovy
index c60308d353..95d5df128a 100644
--- a/modules/nextflow/src/test/groovy/nextflow/ast/DslCodeVisitorTest.groovy
+++ b/modules/nextflow/src/test/groovy/nextflow/ast/NextflowDSLImplTest.groovy
@@ -13,7 +13,7 @@ import test.MockExecutorFactory
  *
  * @author Paolo Di Tommaso 
  */
-class DslCodeVisitorTest extends Dsl2Spec {
+class NextflowDSLImplTest extends Dsl2Spec {
 
     def createCompilerConfig() {
         def config = new CompilerConfiguration()
diff --git a/modules/nextflow/src/test/groovy/nextflow/ast/OperatorXformTest.groovy b/modules/nextflow/src/test/groovy/nextflow/ast/OpXformTest.groovy
similarity index 98%
rename from modules/nextflow/src/test/groovy/nextflow/ast/OperatorXformTest.groovy
rename to modules/nextflow/src/test/groovy/nextflow/ast/OpXformTest.groovy
index c04eed7dc8..5efd9ad51e 100644
--- a/modules/nextflow/src/test/groovy/nextflow/ast/OperatorXformTest.groovy
+++ b/modules/nextflow/src/test/groovy/nextflow/ast/OpXformTest.groovy
@@ -29,12 +29,12 @@ import org.codehaus.groovy.control.customizers.ASTTransformationCustomizer
  *
  * @author Paolo Di Tommaso 
  */
-class OperatorXformTest extends Specification {
+class OpXformTest extends Specification {
 
     private TokenBranchDef eval_branch(String stmt) {
 
         def config = new CompilerConfiguration()
-        config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowXform))
+        config.addCompilationCustomizers( new ASTTransformationCustomizer(OpXform))
 
         def shell = new GroovyShell(config)
         def result = shell.evaluate("""
diff --git a/modules/nextflow/src/test/groovy/nextflow/script/IncludeDefTest.groovy b/modules/nextflow/src/test/groovy/nextflow/script/IncludeDefTest.groovy
index ab81df37e4..01b1b83164 100644
--- a/modules/nextflow/src/test/groovy/nextflow/script/IncludeDefTest.groovy
+++ b/modules/nextflow/src/test/groovy/nextflow/script/IncludeDefTest.groovy
@@ -7,7 +7,7 @@ import spock.lang.Unroll
 import java.nio.file.NoSuchFileException
 import java.nio.file.Path
 
-import nextflow.ast.NextflowXform
+import nextflow.ast.NextflowDSL
 import nextflow.exception.IllegalModulePath
 import nextflow.file.FileHelper
 import org.codehaus.groovy.control.CompilerConfiguration
@@ -174,7 +174,7 @@ class IncludeDefTest extends Specification {
         def binding = new ScriptBinding([params: [foo:1, bar:2]])
         def config = new CompilerConfiguration()
         config.setScriptBaseClass(TestScript.class.name)
-        config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowXform))
+        config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowDSL))
 
         when:
         def script = (TestScript)new GroovyShell(binding, config).parse(INCLUDE)
diff --git a/modules/nextflow/src/test/groovy/nextflow/script/WorkflowDefTest.groovy b/modules/nextflow/src/test/groovy/nextflow/script/WorkflowDefTest.groovy
index 4aa8687bf9..00a0f9dedf 100644
--- a/modules/nextflow/src/test/groovy/nextflow/script/WorkflowDefTest.groovy
+++ b/modules/nextflow/src/test/groovy/nextflow/script/WorkflowDefTest.groovy
@@ -6,7 +6,7 @@ import groovy.util.logging.Slf4j
 import groovyx.gpars.dataflow.DataflowQueue
 import groovyx.gpars.dataflow.DataflowVariable
 import nextflow.Session
-import nextflow.ast.NextflowXform
+import nextflow.ast.NextflowDSL
 import org.codehaus.groovy.control.CompilerConfiguration
 import org.codehaus.groovy.control.MultipleCompilationErrorsException
 import org.codehaus.groovy.control.customizers.ASTTransformationCustomizer
@@ -50,7 +50,7 @@ class WorkflowDefTest extends Dsl2Spec {
         given:
         def config = new CompilerConfiguration()
         config.setScriptBaseClass(TestScript.class.name)
-        config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowXform))
+        config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowDSL))
 
         def SCRIPT = '''
                     
@@ -137,7 +137,7 @@ class WorkflowDefTest extends Dsl2Spec {
         given:
         def config = new CompilerConfiguration()
         config.setScriptBaseClass(TestScript.class.name)
-        config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowXform))
+        config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowDSL))
 
         def SCRIPT = '''
                     
@@ -167,7 +167,7 @@ class WorkflowDefTest extends Dsl2Spec {
         given:
         def config = new CompilerConfiguration()
         config.setScriptBaseClass(TestScript.class.name)
-        config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowXform))
+        config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowDSL))
 
         def SCRIPT = '''
                     
@@ -192,7 +192,7 @@ class WorkflowDefTest extends Dsl2Spec {
         given:
         def config = new CompilerConfiguration()
         config.setScriptBaseClass(TestScript.class.name)
-        config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowXform))
+        config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowDSL))
 
         def SCRIPT = '''
                     
@@ -214,7 +214,7 @@ class WorkflowDefTest extends Dsl2Spec {
         given:
         def config = new CompilerConfiguration()
         config.setScriptBaseClass(TestScript.class.name)
-        config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowXform))
+        config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowDSL))
 
         def SCRIPT = '''
                     
@@ -318,7 +318,7 @@ class WorkflowDefTest extends Dsl2Spec {
         given:
         def config = new CompilerConfiguration()
         config.setScriptBaseClass(TestScript.class.name)
-        config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowXform))
+        config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowDSL))
 
         def SCRIPT = '''
                     
@@ -351,7 +351,7 @@ class WorkflowDefTest extends Dsl2Spec {
         given:
         def config = new CompilerConfiguration()
         config.setScriptBaseClass(TestScript.class.name)
-        config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowXform))
+        config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowDSL))
 
         def SCRIPT = '''
             workflow foo { } 
diff --git a/modules/nextflow/src/test/groovy/nextflow/script/params/ParamsDsl2Test.groovy b/modules/nextflow/src/test/groovy/nextflow/script/params/ParamsDsl2Test.groovy
index a710fefd5b..b703b22187 100644
--- a/modules/nextflow/src/test/groovy/nextflow/script/params/ParamsDsl2Test.groovy
+++ b/modules/nextflow/src/test/groovy/nextflow/script/params/ParamsDsl2Test.groovy
@@ -1,7 +1,7 @@
 package nextflow.script.params
 
 import nextflow.Session
-import nextflow.ast.NextflowXform
+import nextflow.ast.NextflowDSL
 import nextflow.script.BaseScript
 import nextflow.script.ScriptBinding
 import nextflow.script.ScriptMeta
@@ -114,7 +114,7 @@ class ParamsDsl2Test extends Dsl2Spec {
         and:
         def config = new CompilerConfiguration()
         config.setScriptBaseClass(BaseScript.class.name)
-        config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowXform))
+        config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowDSL))
 
         def SCRIPT = '''
                     
@@ -155,7 +155,7 @@ class ParamsDsl2Test extends Dsl2Spec {
         and:
         def config = new CompilerConfiguration()
         config.setScriptBaseClass(BaseScript.class.name)
-        config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowXform))
+        config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowDSL))
 
         def SCRIPT = '''
                     

From c300f0066cf56adb5d71b1a011f24f103d792109 Mon Sep 17 00:00:00 2001
From: Ben Sherman 
Date: Wed, 13 Dec 2023 15:12:59 -0600
Subject: [PATCH 22/36] Minor edits

Signed-off-by: Ben Sherman 
---
 .../groovy/nextflow/processor/TaskOutputCollector.groovy     | 2 +-
 .../src/main/groovy/nextflow/script/BaseScript.groovy        | 2 --
 .../src/main/groovy/nextflow/script/ProcessFactory.groovy    | 5 ++---
 3 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskOutputCollector.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskOutputCollector.groovy
index fe6ddbdc7c..9ca8c0747d 100644
--- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskOutputCollector.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskOutputCollector.groovy
@@ -91,7 +91,7 @@ class TaskOutputCollector implements Map {
      * Get the standard output from the task environment.
      */
     Object stdout() {
-        final result = task.@stdout
+        final result = task.getStdout()
 
         if( result == null && task.type == ScriptType.SCRIPTLET )
             throw new IllegalArgumentException("Missing 'stdout' for process > ${task.lazyName()}")
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy b/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy
index 14e7e9708c..ffe3cd2526 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy
@@ -17,7 +17,6 @@
 package nextflow.script
 
 import java.lang.reflect.InvocationTargetException
-import java.lang.reflect.Method
 import java.nio.file.Paths
 
 import groovy.transform.CompileStatic
@@ -25,7 +24,6 @@ import groovy.util.logging.Slf4j
 import nextflow.NextflowMeta
 import nextflow.Session
 import nextflow.exception.AbortOperationException
-import nextflow.script.dsl.ProcessBuilder
 import nextflow.script.dsl.ProcessDsl
 import nextflow.script.dsl.WorkflowBuilder
 /**
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessFactory.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessFactory.groovy
index e05cfd5dcf..22918eb244 100755
--- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessFactory.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessFactory.groovy
@@ -81,7 +81,6 @@ class ProcessFactory {
      * @return
      *      The {@code Processor} instance
      */
-    @Deprecated
     TaskProcessor createProcessor( String name, Closure body ) {
         assert body
         assert config.process instanceof Map
@@ -90,8 +89,8 @@ class ProcessFactory {
         // Invoke the code block which will return the script closure to the executed.
         // As side effect will set all the property declarations in the 'taskConfig' object.
         final copy = (Closure)body.clone()
-        copy.delegate = builder
-        copy.resolveStrategy = Closure.DELEGATE_FIRST
+        copy.setResolveStrategy(Closure.DELEGATE_FIRST)
+        copy.setDelegate(builder)
         final script = copy.call()
         if ( !script )
             throw new IllegalArgumentException("Missing script in the specified process block -- make sure it terminates with the script string to be executed")

From ce2de32e6a1107e75a6d1bfc82fd3eefa2871cca Mon Sep 17 00:00:00 2001
From: Ben Sherman 
Date: Wed, 13 Dec 2023 15:19:05 -0600
Subject: [PATCH 23/36] Minor edits

Signed-off-by: Ben Sherman 
---
 .../src/test/groovy/nextflow/ast/NextflowDSLImplTest.groovy     | 2 +-
 .../nextflow/src/test/groovy/nextflow/ast/OpXformTest.groovy    | 2 +-
 .../test/groovy/nextflow/processor/PathArityAwareTest.groovy    | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/modules/nextflow/src/test/groovy/nextflow/ast/NextflowDSLImplTest.groovy b/modules/nextflow/src/test/groovy/nextflow/ast/NextflowDSLImplTest.groovy
index 95d5df128a..a8c89e4266 100644
--- a/modules/nextflow/src/test/groovy/nextflow/ast/NextflowDSLImplTest.groovy
+++ b/modules/nextflow/src/test/groovy/nextflow/ast/NextflowDSLImplTest.groovy
@@ -18,7 +18,7 @@ class NextflowDSLImplTest extends Dsl2Spec {
     def createCompilerConfig() {
         def config = new CompilerConfiguration()
         config.setScriptBaseClass(BaseScript.class.name)
-        config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowXform))
+        config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowDSL))
 
         return config
     }
diff --git a/modules/nextflow/src/test/groovy/nextflow/ast/OpXformTest.groovy b/modules/nextflow/src/test/groovy/nextflow/ast/OpXformTest.groovy
index 5efd9ad51e..f4ea31f681 100644
--- a/modules/nextflow/src/test/groovy/nextflow/ast/OpXformTest.groovy
+++ b/modules/nextflow/src/test/groovy/nextflow/ast/OpXformTest.groovy
@@ -52,7 +52,7 @@ class OpXformTest extends Specification {
     private TokenMultiMapDef eval_multiMap(String stmt) {
 
         def config = new CompilerConfiguration()
-        config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowXform))
+        config.addCompilationCustomizers( new ASTTransformationCustomizer(OpXform))
 
         def shell = new GroovyShell(config)
         def result = shell.evaluate("""
diff --git a/modules/nextflow/src/test/groovy/nextflow/processor/PathArityAwareTest.groovy b/modules/nextflow/src/test/groovy/nextflow/processor/PathArityAwareTest.groovy
index 0dd03ef6f9..15d4723184 100644
--- a/modules/nextflow/src/test/groovy/nextflow/processor/PathArityAwareTest.groovy
+++ b/modules/nextflow/src/test/groovy/nextflow/processor/PathArityAwareTest.groovy
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package nextflow.processor
+package nextflow.script
 
 import spock.lang.Specification
 import spock.lang.Unroll

From 47a85beb4399f9bb9d7ab13c0cd02fd1b28ee956 Mon Sep 17 00:00:00 2001
From: Ben Sherman 
Date: Sat, 16 Dec 2023 18:19:46 -0600
Subject: [PATCH 24/36] Fix storeDir warning and task context caching

Signed-off-by: Ben Sherman 
---
 .../src/main/groovy/nextflow/cache/CacheDB.groovy         | 5 +++--
 .../main/groovy/nextflow/processor/TaskProcessor.groovy   | 8 +++++++-
 .../src/main/groovy/nextflow/processor/TaskRun.groovy     | 8 ++++++++
 3 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/modules/nextflow/src/main/groovy/nextflow/cache/CacheDB.groovy b/modules/nextflow/src/main/groovy/nextflow/cache/CacheDB.groovy
index 0238af186b..214e3235e3 100644
--- a/modules/nextflow/src/main/groovy/nextflow/cache/CacheDB.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/cache/CacheDB.groovy
@@ -139,8 +139,9 @@ class CacheDB implements Closeable {
         final proc = task.processor
         final key = task.hash
 
-        // -- save the task context only if caching is enabled for the process
-        TaskContext ctx = proc.isCacheable() ? task.context : null
+        // save the context map for caching purpose
+        // only the 'cache' is active and
+        TaskContext ctx = proc.isCacheable() && task.hasCacheableValues() ? task.context : null
 
         def record = new ArrayList(3)
         record[0] = trace.serialize()
diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy
index 2a39ae4798..99204ba94a 100644
--- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy
@@ -638,6 +638,12 @@ class TaskProcessor {
             return false
         }
 
+        // -- when store path is set, only output params of type 'file' can be specified
+        if( task.inputFiles.size() == 0 ) {
+            checkWarn "[${safeTaskName(task)}] StoreDir can only be used when using 'file' outputs"
+            return false
+        }
+
         if( !task.config.getStoreDir().exists() ) {
             log.trace "[${safeTaskName(task)}] Store dir does not exists > ${task.config.storeDir} -- return false"
             // no folder -> no cached result
@@ -705,7 +711,7 @@ class TaskProcessor {
             return false
         }
 
-        if( !entry.context ) {
+        if( task.hasCacheableValues() && !entry.context ) {
             log.trace "[${safeTaskName(task)}] Missing cache context -- return false"
             return false
         }
diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy
index 880ad24ea7..628f7c7e19 100644
--- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy
@@ -392,6 +392,14 @@ class TaskRun implements Cloneable {
             : getScript()
     }
 
+
+    /**
+     * Check whenever there are values to be cached
+     */
+    boolean hasCacheableValues() {
+        return body.type != ScriptType.SCRIPTLET
+    }
+
     /**
      * @return A map object containing all the task input files as  pairs
      */

From cc2c08e19e54aa8341e59e1f41214878fe9e8318 Mon Sep 17 00:00:00 2001
From: Ben Sherman 
Date: Sat, 16 Dec 2023 19:20:59 -0600
Subject: [PATCH 25/36] Merge upstream changes

Signed-off-by: Ben Sherman 
---
 Makefile                                      |   3 +-
 VERSION                                       |   2 +-
 build.gradle                                  |  27 +-
 changelog.txt                                 |  59 ++
 docker/Dockerfile                             |   2 +-
 docs/amazons3.md                              |  12 +-
 docs/aws.md                                   |  60 +-
 docs/channel.md                               | 234 ++++--
 docs/conf.py                                  |   4 +-
 docs/config.md                                |  99 ++-
 docs/container.md                             |   2 +-
 docs/executor.md                              |   2 +
 docs/fusion.md                                |  19 +-
 docs/mail.md                                  |   2 +-
 docs/operator.md                              | 715 +++++++++---------
 docs/process.md                               |  89 ++-
 docs/script.md                                |  46 +-
 .../snippets/{transpose.nf => transpose-1.nf} |   0
 .../{transpose.out => transpose-1.out}        |   0
 docs/snippets/transpose-2-with-remainder.nf   |   7 +
 docs/snippets/transpose-2-with-remainder.out  |   6 +
 docs/snippets/transpose-2.nf                  |   7 +
 docs/snippets/transpose-2.out                 |   5 +
 docs/workflow.md                              |   4 +-
 modules/nextflow/build.gradle                 |   4 +-
 .../src/main/groovy/nextflow/Channel.groovy   |   5 +
 .../src/main/groovy/nextflow/NF.groovy        |   4 +
 .../main/groovy/nextflow/NextflowMeta.groovy  |  17 +-
 .../src/main/groovy/nextflow/Session.groovy   |  12 +-
 .../nextflow/ast/NextflowDSLImpl.groovy       |  10 +-
 .../main/groovy/nextflow/cli/CmdInfo.groovy   |   6 +-
 .../main/groovy/nextflow/cli/CmdRun.groovy    |   6 +-
 .../main/groovy/nextflow/cli/Launcher.groovy  |  18 +-
 .../nextflow/container/ContainerConfig.groovy |   2 +
 .../container/SingularityCache.groovy         |   2 +-
 .../resolver/DefaultContainerResolver.groovy  |  14 +-
 .../nextflow/executor/GridTaskHandler.groovy  |   2 +-
 .../main/groovy/nextflow/extension/CH.groovy  |  67 +-
 .../nextflow/extension/GroupTupleOp.groovy    |   2 +-
 .../groovy/nextflow/extension/MixOp.groovy    |  16 +-
 .../groovy/nextflow/file/FileCollector.groovy |   2 +-
 .../nextflow/fusion/FusionConfig.groovy       |   6 +
 .../nextflow/fusion/FusionEnvProvider.groovy  |   2 +
 .../nextflow/processor/PublishDir.groovy      |  76 +-
 .../nextflow/processor/TaskProcessor.groovy   |   2 +-
 .../groovy/nextflow/processor/TaskRun.groovy  |   2 +-
 .../groovy/nextflow/script/ProcessDef.groovy  |  12 +-
 .../nextflow/script/ProcessOutput.groovy      |  29 +-
 .../nextflow/script/ScriptBinding.groovy      |   6 +-
 .../nextflow/script/WorkflowMetadata.groovy   |   8 +-
 .../nextflow/script/WorkflowNotifier.groovy   |  12 +-
 .../resources/META-INF/build-info.properties  |   4 +
 .../main/resources/META-INF/plugins-info.txt  |  18 +-
 .../groovy/nextflow/NextflowMetaTest.groovy   |  10 +-
 .../groovy/nextflow/cli/CmdCloneTest.groovy   |   2 +
 .../groovy/nextflow/cli/CmdRunTest.groovy     |   2 +-
 .../container/ContainerConfigTest.groovy      |   3 +
 .../nextflow/extension/PublishOpTest.groovy   |   1 +
 .../nextflow/fusion/FusionConfigTest.groovy   |  17 +-
 .../nextflow/processor/PublishDirTest.groovy  |   4 +-
 .../scm/GiteaRepositoryProviderTest.groovy    |   2 +
 .../nextflow/script/ScriptBindingTest.groovy  |  10 +-
 .../script/WorkflowMetadataTest.groovy        |   9 +-
 .../script/WorkflowNotifierTest.groovy        |  37 +-
 .../script/params/ParamsOutTest.groovy        |  83 +-
 modules/nf-commons/build.gradle               |   2 +-
 .../src/main/nextflow/BuildInfo.groovy        |  88 +++
 .../nf-commons/src/main/nextflow/Const.groovy |  64 +-
 .../plugin/CustomPluginManager.groovy         |   2 +-
 .../main/nextflow/plugin/PluginUpdater.groovy |   2 +-
 .../main/nextflow/plugin/PluginsFacade.groovy |   2 +-
 .../src/main/nextflow/util/StringUtils.groovy |   2 +-
 .../src/test/nextflow/BuildInfoTest.groovy    |  35 +
 .../nextflow/plugin/PluginsFacadeTest.groovy  |  13 +
 .../test/nextflow/util/StringUtilsTest.groovy |  16 +-
 .../plugin/TestPluginDescriptorFinder.groovy  |  16 +-
 nextflow                                      |   2 +-
 nextflow.md5                                  |   2 +-
 nextflow.sha1                                 |   2 +-
 nextflow.sha256                               |   2 +-
 packing.gradle                                |   4 +-
 plugins/nf-amazon/build.gradle                |  22 +-
 plugins/nf-amazon/changelog.txt               |   7 +
 .../cloud/aws/batch/AwsBatchExecutor.groovy   |   2 +-
 .../aws/batch/AwsBatchTaskHandler.groovy      | 124 ++-
 .../cloud/aws/batch/AwsOptions.groovy         |  13 +
 .../cloud/aws/config/AwsBatchConfig.groovy    |  29 +-
 .../cloud/aws/fusion/AwsFusionEnv.groovy      |   7 +
 .../cloud/aws/nio/S3FileSystemProvider.java   |   8 +-
 .../aws/nio/util/S3ObjectSummaryLookup.java   |   4 +-
 .../nextflow/cloud/aws/util/S3BashLib.groovy  |  57 +-
 .../src/resources/META-INF/MANIFEST.MF        |   4 +-
 .../aws/batch/AwsBatchTaskHandlerTest.groovy  | 178 ++++-
 .../cloud/aws/batch/AwsOptionsTest.groovy     |  21 +
 .../aws/config/AwsBatchConfigTest.groovy      |  27 +
 .../cloud/aws/fusion/AwsFusionEnvTest.groovy  |  24 +
 .../cloud/aws/util/S3BashLibTest.groovy       |  76 ++
 .../processor/PublishDirS3Test.groovy         |  34 +-
 plugins/nf-azure/build.gradle                 |   2 +-
 plugins/nf-azure/changelog.txt                |   4 +
 .../cloud/azure/batch/AzBatchService.groovy   |  26 +-
 .../cloud/azure/config/AzPoolOpts.groovy      |   3 +
 .../src/resources/META-INF/MANIFEST.MF        |   4 +-
 .../azure/batch/AzBatchServiceTest.groovy     |  14 +-
 .../cloud/azure/config/AzPoolOptsTest.groovy  |  94 +++
 plugins/nf-cloudcache/build.gradle            |   2 +-
 plugins/nf-cloudcache/changelog.txt           |   3 +
 .../src/resources/META-INF/MANIFEST.MF        |   4 +-
 plugins/nf-codecommit/build.gradle            |   4 +-
 plugins/nf-codecommit/changelog.txt           |   4 +
 .../src/resources/META-INF/MANIFEST.MF        |   4 +-
 plugins/nf-console/build.gradle               |   2 +-
 .../src/resources/META-INF/MANIFEST.MF        |   4 +-
 plugins/nf-ga4gh/build.gradle                 |   2 +-
 .../src/resources/META-INF/MANIFEST.MF        |   4 +-
 plugins/nf-google/build.gradle                |   2 +-
 plugins/nf-google/changelog.txt               |   9 +
 .../nextflow/cloud/google/GoogleOpts.groovy   |  19 +-
 .../GoogleBatchMachineTypeSelector.groovy     |   5 +-
 .../batch/GoogleBatchTaskHandler.groovy       |  19 +
 .../google/batch/client/BatchConfig.groovy    |   7 +-
 .../google/config/GoogleRetryOpts.groovy      |  44 ++
 .../google/config/GoogleStorageOpts.groovy    |  32 +
 .../cloud/google/util/GsPathFactory.groovy    |  23 +-
 .../src/resources/META-INF/MANIFEST.MF        |   4 +-
 .../batch/GoogleBatchTaskHandlerTest.groovy   |  20 +-
 .../google/config/GoogleRetryOptsTest.groovy} |  31 +-
 .../GoogleLifeSciencesHelperTest.groovy       |   6 +-
 .../GoogleLifeSciencesTaskHandlerTest.groovy  |  22 +-
 .../google/util/GsPathFactoryTest.groovy      |  25 +
 plugins/nf-tower/build.gradle                 |   2 +-
 plugins/nf-tower/changelog.txt                |   4 +
 .../src/resources/META-INF/MANIFEST.MF        |   4 +-
 plugins/nf-wave/build.gradle                  |   2 +-
 plugins/nf-wave/changelog.txt                 |   9 +
 .../io/seqera/wave/plugin/WaveClient.groovy   |  22 +
 .../io/seqera/wave/plugin/WaveFactory.groovy  |  32 +-
 .../io/seqera/wave/plugin/WaveObserver.groovy |  85 ---
 .../resolver/WaveContainerResolver.groovy     |  12 +-
 .../src/resources/META-INF/MANIFEST.MF        |   4 +-
 .../seqera/wave/plugin/WaveClientTest.groovy  |  21 +-
 .../seqera/wave/plugin/WaveFactoryTest.groovy |  16 +
 .../resolver/WaveContainerResolverTest.groovy |   2 +-
 tests/checks/fusion-symlink.nf/.checks        |  25 +
 tests/checks/fusion-symlink.nf/.config        |   4 +
 tests/checks/fusion-symlink.nf/.expected      |   1 +
 tests/checks/topic-channel.nf/.checks         |  15 +
 tests/checks/topic-channel.nf/.expected       |   2 +
 tests/fusion-symlink.nf                       |  61 ++
 tests/topic-channel.nf                        |  37 +
 validation/awsbatch.sh                        |   7 +
 validation/awsfargate.config                  |  14 +
 validation/google.sh                          |  11 +-
 validation/google_credentials.gpg             | Bin 1726 -> 0 bytes
 154 files changed, 2701 insertions(+), 982 deletions(-)
 rename docs/snippets/{transpose.nf => transpose-1.nf} (100%)
 rename docs/snippets/{transpose.out => transpose-1.out} (100%)
 create mode 100644 docs/snippets/transpose-2-with-remainder.nf
 create mode 100644 docs/snippets/transpose-2-with-remainder.out
 create mode 100644 docs/snippets/transpose-2.nf
 create mode 100644 docs/snippets/transpose-2.out
 create mode 100644 modules/nextflow/src/main/resources/META-INF/build-info.properties
 create mode 100644 modules/nf-commons/src/main/nextflow/BuildInfo.groovy
 create mode 100644 modules/nf-commons/src/test/nextflow/BuildInfoTest.groovy
 create mode 100644 plugins/nf-azure/src/test/nextflow/cloud/azure/config/AzPoolOptsTest.groovy
 create mode 100644 plugins/nf-google/src/main/nextflow/cloud/google/config/GoogleRetryOpts.groovy
 create mode 100644 plugins/nf-google/src/main/nextflow/cloud/google/config/GoogleStorageOpts.groovy
 rename plugins/{nf-wave/src/test/io/seqera/wave/plugin/WaveObserverTest.groovy => nf-google/src/test/nextflow/cloud/google/config/GoogleRetryOptsTest.groovy} (51%)
 delete mode 100644 plugins/nf-wave/src/main/io/seqera/wave/plugin/WaveObserver.groovy
 create mode 100644 tests/checks/fusion-symlink.nf/.checks
 create mode 100644 tests/checks/fusion-symlink.nf/.config
 create mode 100644 tests/checks/fusion-symlink.nf/.expected
 create mode 100644 tests/checks/topic-channel.nf/.checks
 create mode 100644 tests/checks/topic-channel.nf/.expected
 create mode 100644 tests/fusion-symlink.nf
 create mode 100644 tests/topic-channel.nf
 create mode 100644 validation/awsfargate.config
 delete mode 100644 validation/google_credentials.gpg

diff --git a/Makefile b/Makefile
index 6d21ea1548..bf55192bd3 100644
--- a/Makefile
+++ b/Makefile
@@ -33,6 +33,7 @@ clean:
 	rm -rf modules/nextflow/.nextflow*
 	rm -rf modules/nextflow/work
 	rm -rf build
+	rm -rf buildSrc/build
 	rm -rf modules/*/build
 	rm -rf plugins/*/build
 	./gradlew clean
@@ -42,7 +43,7 @@ compile:
 	@echo "DONE `date`"
 
 assemble:
-	./gradlew compile assemble
+	./gradlew buildInfo compile assemble
 
 check:
 	./gradlew check
diff --git a/VERSION b/VERSION
index 7d7a070243..c412fc2d77 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-23.10.0
+23.11.0-edge
diff --git a/build.gradle b/build.gradle
index fa5df23a8d..0c13c17f78 100644
--- a/build.gradle
+++ b/build.gradle
@@ -48,8 +48,15 @@ def projects(String...args) {
     args.collect {project(it)}
 }
 
+String gitVersion() {
+    def p = new ProcessBuilder() .command('sh','-c','git rev-parse --short HEAD') .start()
+    def r = p.waitFor()
+    return r==0 ? p.text.trim() : '(unknown)'
+}
+
 group = 'io.nextflow'
 version = rootProject.file('VERSION').text.trim()
+ext.commitId = gitVersion()
 
 allprojects {
     apply plugin: 'java'
@@ -190,20 +197,23 @@ jar.enabled = false
  */
 task buildInfo { doLast {
 
-    def file0 = file('./modules/nf-commons/src/main/nextflow/Const.groovy')
+    def file0 = file('modules/nextflow/src/main/resources/META-INF/build-info.properties')
     def buildNum = 0
     def src = file0.text
-    src.find(/APP_BUILDNUM *= *([0-9]*)/) { buildNum = it[1]?.toInteger()+1 }
-    src = src.replaceAll('APP_VER *= *"[0-9a-zA-Z_\\-\\.]+"', "APP_VER = \"${version}\"" as String)
-    src = src.replaceAll('APP_TIMESTAMP *= *[0-9]*', "APP_TIMESTAMP = ${System.currentTimeMillis()}" as String)
-    if( buildNum ) {
-        src = src.replaceAll('APP_BUILDNUM *= *[0-9]*', "APP_BUILDNUM = ${buildNum}" as String)
-    }
-    else {
+    src.find(/build *= *([0-9]*)/) { buildNum = it[1]?.toInteger()+1 }
+    if( !buildNum ) {
         println "WARN: Unable to find current build number"
     }
     file0.text = src
 
+    // -- update build-info file
+    file0.text = """\
+    build=${buildNum}
+    version=${version}
+    timestamp=${System.currentTimeMillis()}
+    commitId=${project.property('commitId')}
+    """.stripIndent()
+    
     // -- update 'nextflow' wrapper
     file0 = file('nextflow')
     src = file0.text
@@ -238,7 +248,6 @@ task buildInfo { doLast {
  * Compile sources and copies all libs to target directory
  */
 task compile {
-    dependsOn buildInfo
     dependsOn allprojects.classes
 }
 
diff --git a/changelog.txt b/changelog.txt
index ac135b9f76..5ce78b9124 100644
--- a/changelog.txt
+++ b/changelog.txt
@@ -1,5 +1,64 @@
 NEXTFLOW CHANGE-LOG
 ===================
+23.11.0-edge - 24 Nov 2023
+- Add `fusion.cacheSize` config option (#4518) [2faadc22]
+- Add Topic channel type (experimental) (#4459) [921313d1]
+- Add Google Batch native retry on spot termination (#4500) [ea1c1b70]
+- Add Retry policy to Google Storage (#4524) [c271bb18]
+- Add ability detect Google Batch spot interruption (#4462) [d49f02ae]
+- Add doc tests, move some snippets to separate files (#3959) [0ff3b305]
+- Add docs section on container requirements (#4501) [3fb29f78]
+- Add labels field in Job request for Google Batch (#4538) [627c595e]
+- Add note about limitations of glacier auto retrieval (#4514) [82e56799]
+- Add note about local executor and enforcing resource limits (#4468) [6a0626f7]
+- Add section about sharing modules (#4482) [3e66fba4]
+- Add section on process directives to plugin docs (#4477) [d9ee9870]
+- Add support for Azure low-priority pool (#4527) [8320ea10]
+- Add support for FUSION_AWS_REGION (#4481) [8f8b09fa]
+- Add support for Fusion when using Singularity OCI mode (#4508) [4f3aa631]
+- Add support for K8s schedulerName pod spec (#4485) [ci fast] [dfc7b7c8]
+- Add support for Singularity OCI mode (#4440) [f5362a7b]
+- Allow the use of error built-in function in onComplete handler (#4458) [ci fast] [35a4424b]
+- Fix Bug in JsonSplitter ordering [ci fast] [8ec14dd2]
+- Fix Bypass Google Batch Price query if task cpus and memory are defined (#4521) [7f8f20d3]
+- Fix Checkout remote tag if checkout remote branch fails (#4247) [b8907ccb]
+- Fix Fusion symlinks when publishing files (#4348) [89f09fe0]
+- Fix Inspect command fails with Singularity [f5bb829f]
+- Fix ParamsMap copyWith param aliases (#4188) [b480ee0e]
+- Fix Singularity docs [e952299f]
+- Fix container hashing for Singularity + Wave containers [4c6f2e85]
+- Fix detection of Conda local path made by Wave client [ci fast] (#4532) [4d5bc216]
+- Fix doc tests to fail on test failure (#4505) [4d326551]
+- Fix errors when NXF_HOME contains spaces (#4456) [ci fast] [fe5bea99]
+- Fix Google Batch network/subnetwork docs (#4475) [27d132f3]
+- Fix rounding error with long durations (#4496) [ci fast] [0356178b]
+- Fix security vulnerabilities (#4513) [a310c777]
+- Fix Use consistently NXF_TASK_WORKDIR (#4484) [48ee3c64]
+- Improve error details for AbortOperationException [35609cb0]
+- Improve operator docs (#4502) [38210e11]
+- Makefile clean to also remove buildSrc/build (#4517) [2ccb05d0]
+- Minor test improvements [ci fast] [171831ea]
+- Minor types improvement for mix operator [ci fast] [91c1ab15]
+- Normalise channel docs [b641d677]
+- Remove deprecated TowerArchiver feature [ff8e06a3]
+- Remove dsl1 deprecated code (part 2) [159effb1]
+- Remove dsl1 deprecated code [2b433a52]
+- Remove incorrect note about workflow inputs (#4509) [54bc0b7d]
+- Return error if plugin version is not specified in offline mode (#4487) [f5d7246e]
+- Update README.md with new branding color for Nextflow (#4412) [7a13b18b]
+- Update background color of docs status badges (#4411) [3cb1c53c]
+- Update logging filter for Google Batch provider. (#4488) [66a3ed19]
+- Bump Gradle 8.4 and test vs Java 21 (#4450) [8cb2702c]
+- Bump nf-amazon@2.2.0 [8e2d7879]
+- Bump nf-azure@1.4.0 [7c47d090]
+- Bump nf-cloudcache@0.3.1 [65240b75]
+- Bump nf-codecommit@0.1.6 [725f0510]
+- Bump nf-console@1.0.7 [a307686c]
+- Bump nf-ga4gh@1.1.1 [e54ea007]
+- Bump nf-google@1.9.0 [033ec92c]
+- Bump nf-tower@1.7.0 [836a44a5]
+- Bump nf-wave@1.1.0 [620523ef]
+
 23.10.0 - 15 Oct 2023
 - Add support for K8s hostPath [10c32325]
 - Add AWS SES docs [b83e7148]
diff --git a/docker/Dockerfile b/docker/Dockerfile
index de56944d02..e016583e98 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -1,4 +1,4 @@
-FROM amazoncorretto:17.0.8
+FROM amazoncorretto:17.0.9
 RUN yum install -y procps-ng shadow-utils which
 
 ENV NXF_HOME=/.nextflow
diff --git a/docs/amazons3.md b/docs/amazons3.md
index bfcabb23c9..41bcb407fa 100644
--- a/docs/amazons3.md
+++ b/docs/amazons3.md
@@ -1,8 +1,8 @@
 (amazons3-page)=
 
-# Amazon S3 storage
+# AWS S3 storage
 
-Nextflow includes support for Amazon S3 storage. Files stored in an S3 bucket can be accessed transparently in your pipeline script like any other file in the local file system.
+Nextflow includes support for AWS S3 storage. Files stored in an S3 bucket can be accessed transparently in your pipeline script like any other file in the local file system.
 
 ## S3 path
 
@@ -24,10 +24,10 @@ See the {ref}`script-file-io` section to learn more about available file operati
 
 ## Security credentials
 
-Amazon access credentials can be provided in two ways:
+AWS access credentials can be provided in two ways:
 
 1. Using AWS access and secret keys in your pipeline configuration.
-2. Using IAM roles to grant access to S3 storage on Amazon EC2 instances.
+2. Using IAM roles to grant access to S3 storage on AWS EC2 instances.
 
 ### AWS access and secret keys
 
@@ -52,13 +52,13 @@ If the access credentials are not found in the above file, Nextflow looks for AW
 
 More information regarding [AWS Security Credentials](http://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html) are available in the AWS documentation.
 
-### IAM roles with Amazon EC2 instances
+### IAM roles with AWS EC2 instances
 
 When running your pipeline in an EC2 instance, IAM roles can be used to grant access to AWS resources.
 
 In this scenario, you only need to launch the EC2 instance with an IAM role which includes the `AmazonS3FullAccess` policy. Nextflow will detect and automatically acquire the permission to access S3 storage, without any further configuration.
 
-Learn more about [Using IAM Roles to Delegate Permissions to Applications that Run on Amazon EC2](http://docs.aws.amazon.com/IAM/latest/UserGuide/roles-usingrole-ec2instance.html) in the Amazon documentation.
+Learn more about [Using IAM Roles to Delegate Permissions to Applications that Run on AWS EC2](http://docs.aws.amazon.com/IAM/latest/UserGuide/roles-usingrole-ec2instance.html) in the AWS documentation.
 
 ## China regions
 
diff --git a/docs/aws.md b/docs/aws.md
index 4cd2488f1b..acd6bbd01b 100644
--- a/docs/aws.md
+++ b/docs/aws.md
@@ -1,6 +1,6 @@
 (aws-page)=
 
-# Amazon Web Services
+# AWS Cloud
 
 ## AWS security credentials
 
@@ -132,30 +132,24 @@ See the [bucket policy documentation](https://docs.aws.amazon.com/config/latest/
 
 ## AWS Batch
 
-[AWS Batch](https://aws.amazon.com/batch/) is a managed computing service that allows the execution of containerised workloads in the Amazon cloud infrastructure. It dynamically provisions the optimal quantity and type of compute resources (e.g., CPU or memory optimized compute resources) based on the volume and specific resource requirements of the jobs submitted.
+[AWS Batch](https://aws.amazon.com/batch/) is a managed computing service that allows the execution of containerised workloads in the AWS cloud infrastructure. It dynamically provisions the optimal quantity and type of compute resources (e.g., CPU or memory optimized compute resources) based on the volume and specific resource requirements of the jobs submitted.
 
 Nextflow provides built-in support for AWS Batch, allowing the seamless deployment of Nextflow pipelines in the cloud, in which tasks are offloaded as Batch jobs.
 
 Read the {ref}`AWS Batch executor ` section to learn more about the `awsbatch` executor in Nextflow.
 
-(aws-batch-config)=
+(aws-batch-cli)=
 
 ### AWS CLI
 
-Nextflow needs the [AWS command line tool](https://aws.amazon.com/cli/) (`aws`) to be available in the container in which tasks are executed, in order to stage input files and output files to and from S3 storage.
-
 :::{tip}
-When using {ref}`wave-page` and {ref}`fusion-page`, the AWS command line tool is not needed for task containers or the underlying EC2 instances when running Nextflow on AWS Batch. See the {ref}`fusion-page` documentation for more details.
+The need for the AWS CLI is considered a legacy requirement for the deployment of Nextflow pipelines with AWS Batch.
+Instead, consider using {ref}`wave-page` and {ref}`fusion-page` to facilitate access to S3 without using the AWS CLI.
 :::
 
-The `aws` command can be made available in the container in two ways:
-
-1. Installed in the Docker image(s) used during the pipeline execution,
-2. Installed in a custom [AMI (Amazon Machine Image)](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AMIs.html) to use in place of the default AMI when configuring AWS Batch (see next section).
-
-The latter approach is preferred because it allows the use of existing Docker images without having to add the AWS CLI to each one.
+Nextflow uses the [AWS command line tool](https://aws.amazon.com/cli/) (`aws`) to stage input files and output files between S3 and the task containers.
 
-See the sections below to learn how to create a custom AMI and install the AWS CLI tool in it.
+The `aws` command can be made available by either (1) installing it in the container image(s) or (2) installing it in a {ref}`custom AMI ` to be used instead of the default AMI when configuring AWS Batch.
 
 ### Get started
 
@@ -194,7 +188,7 @@ process {
 
 aws {
     batch {
-        // NOTE: this setting is only required if the AWS CLI tool is installed in a custom AMI
+        // NOTE: this setting is only required if the AWS CLI is installed in a custom AMI
         cliPath = '/home/ec2-user/miniconda/bin/aws'
     }
     region = 'us-east-1'
@@ -249,6 +243,8 @@ containerOptions '--ulimit nofile=1280:2560 --ulimit nproc=16:32 --privileged'
 
 Check the [AWS documentation](https://docs.aws.amazon.com/batch/latest/APIReference/API_ContainerProperties.html) for further details.
 
+(aws-custom-ami)=
+
 ## Custom AMI
 
 There are several reasons why you might need to create your own [AMI (Amazon Machine Image)](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AMIs.html) to use in your Compute Environments:
@@ -282,10 +278,11 @@ Any additional software must be installed on the EC2 instance *before* creating
 ### AWS CLI installation
 
 :::{tip}
-When using {ref}`wave-page` and {ref}`fusion-page`, the AWS command line tool is not needed for task containers or the underlying EC2 instances when running Nextflow on AWS Batch. See the {ref}`fusion-page` documentation for more details.
+The need for the AWS CLI is considered a legacy requirement for the deployment of Nextflow pipelines with AWS Batch.
+Instead, consider using {ref}`wave-page` and {ref}`fusion-page` to facilitate access to S3 without using the AWS CLI.
 :::
 
-The [AWS CLI tool](https://aws.amazon.com/cli) should be installed in your custom AMI using a self-contained package manager such as [Conda](https://conda.io). That way, you can control which version of Python is used by the AWS CLI (which is written in Python).
+The [AWS CLI](https://aws.amazon.com/cli) should be installed in your custom AMI using a self-contained package manager such as [Conda](https://conda.io). That way, you can control which version of Python is used by the AWS CLI (which is written in Python).
 
 If you don't use Conda, the `aws` command will attempt to use the version of Python that is installed in the container, and it won't be able to find the necessary dependencies.
 
@@ -410,7 +407,7 @@ The bucket path should include at least a top level directory name, e.g. `s3://m
 
 Nextflow allows the use of multiple executors in the same workflow application. This feature enables the deployment of hybrid workloads in which some jobs are executed in the local computer or local computing cluster and some jobs are offloaded to AWS Batch.
 
-To enable this feature, use one or more {ref}`config-process-selectors` in your Nextflow configuration to apply the AWS Batch {ref}`configuration ` to the subset of processes that you want to offload. For example:
+To enable this feature, use one or more {ref}`config-process-selectors` in your Nextflow configuration to apply the AWS Batch configuration to the subset of processes that you want to offload. For example:
 
 ```groovy
 aws {
@@ -499,6 +496,35 @@ There are multiple reasons why this can happen. They are mainly related to the C
 
 This [AWS page](https://aws.amazon.com/premiumsupport/knowledge-center/batch-job-stuck-runnable-status/) provides several resolutions and tips to investigate and work around the issue.
 
+## AWS Fargate
+
+:::{versionadded} 23.12.0-edge
+:::
+
+Nextflow provides experimental support for the execution of [AWS Batch jobs with Fargate resources](https://docs.aws.amazon.com/batch/latest/userguide/fargate.html).
+
+AWS Fargate is a technology that you can use with AWS Batch to run containers without having to manage servers or  EC2 instances.
+With AWS Fargate, you no longer have to provision, configure, or scale clusters of virtual machines to run containers.
+
+To enable the use of AWS Fargate in your pipeline use the following settings in your `nextflow.config` file:
+
+```groovy
+process.executor = 'awsbatch'
+process.queue = ''
+aws.region = ''
+aws.batch.platformType = 'fargate'
+aws.batch.jobRole = 'JOB ROLE ARN'
+aws.batch.executionRole = 'EXECUTION ROLE ARN'
+wave.enabled = true
+```
+
+See the AWS documentation for details how to create the required AWS Batch queue for Fargate, the Batch Job Role
+and the Batch Execution Role.
+
+:::{note}
+This feature requires the use {ref}`Wave ` container provisioning service.
+:::
+
 ## Advanced configuration
 
 Read the {ref}`AWS configuration` section to learn more about advanced configuration options.
diff --git a/docs/channel.md b/docs/channel.md
index 754d48c4ad..157ae08534 100644
--- a/docs/channel.md
+++ b/docs/channel.md
@@ -19,7 +19,8 @@ In Nextflow there are two kinds of channels: *queue channels* and *value channel
 
 ### Queue channel
 
-A *queue channel* is a non-blocking unidirectional FIFO queue which connects two processes, channel factories, or operators.
+A *queue channel* is a non-blocking unidirectional FIFO queue connecting a *producer* process (i.e. outputting a value)
+to a consumer process, or an operators.
 
 A queue channel can be created by factory methods ([of](#of), [fromPath](#frompath), etc), operators ({ref}`operator-map`, {ref}`operator-flatmap`, etc), and processes (see {ref}`Process outputs `).
 
@@ -27,9 +28,12 @@ A queue channel can be created by factory methods ([of](#of), [fromPath](#frompa
 
 ### Value channel
 
-A *value channel* contains a single value and can be consumed any number of times by a process or operator.
+A *value channel* can be bound (i.e. assigned) with one and only one value, and can be consumed any number of times by
+a process or an operator.
 
-A value channel can be created with the [value](#value) factory method or by any operator that produces a single value ({ref}`operator-first`, {ref}`operator-collect`, {ref}`operator-reduce`, etc). Additionally, a process will emit value channels if it is invoked with all value channels, including simple values which are implicitly wrapped in a value channel.
+A value channel can be created with the [value](#value) factory method or by any operator that produces a single value
+({ref}`operator-first`, {ref}`operator-collect`, {ref}`operator-reduce`, etc). Additionally, a process will emit value
+channels if it is invoked with all value channels, including simple values which are implicitly wrapped in a value channel.
 
 For example:
 
@@ -52,7 +56,8 @@ workflow {
 }
 ```
 
-In the above example, since the `foo` process is invoked with a simple value instead of a channel, the input is implicitly wrapped in a value channel, and the output is also emitted as a value channel.
+In the above example, since the `foo` process is invoked with a simple value instead of a channel, the input is implicitly
+wrapped in a value channel, and the output is also emitted as a value channel.
 
 See also: {ref}`process-multiple-input-channels`.
 
@@ -63,14 +68,15 @@ See also: {ref}`process-multiple-input-channels`.
 Channels may be created explicitly using the following channel factory methods.
 
 :::{versionadded} 20.07.0
-`channel` was introduced as an alias of `Channel`, allowing factory methods to be specified as `channel.of()` or `Channel.of()`, and so on.
+`channel` was introduced as an alias of `Channel`, allowing factory methods to be specified as `channel.of()` or
+`Channel.of()`, and so on.
 :::
 
 (channel-empty)=
 
 ### empty
 
-The `empty` factory method, by definition, creates a channel that doesn't emit any value.
+The `channel.empty` factory method, by definition, creates a channel that doesn't emit any value.
 
 See also: {ref}`operator-ifempty`.
 
@@ -79,13 +85,13 @@ See also: {ref}`operator-ifempty`.
 ### from
 
 :::{deprecated} 19.09.0-edge
-Use [of](#of) or [fromList](#fromlist) instead.
+Use [channel.of](#of) or [channel.fromList](#fromlist) instead.
 :::
 
-The `from` method allows you to create a channel emitting any sequence of values that are specified as the method argument, for example:
+The `channel.from` method allows you to create a channel emitting any sequence of values that are specified as the method argument, for example:
 
 ```groovy
-ch = Channel.from( 1, 3, 5, 7 )
+ch = channel.from( 1, 3, 5, 7 )
 ch.subscribe { println "value: $it" }
 ```
 
@@ -101,25 +107,25 @@ value: 7
 The following example shows how to create a channel from a *range* of numbers or strings:
 
 ```groovy
-zeroToNine = Channel.from( 0..9 )
-strings = Channel.from( 'A'..'Z' )
+zeroToNine = channel.from( 0..9 )
+strings = channel.from( 'A'..'Z' )
 ```
 
 :::{note}
-When the `from` argument is an object implementing the (Java) [Collection](http://docs.oracle.com/javase/7/docs/api/java/util/Collection.html) interface, the resulting channel emits the collection entries as individual items.
+When the `channel.from` argument is an object implementing the (Java) [Collection](http://docs.oracle.com/javase/7/docs/api/java/util/Collection.html) interface, the resulting channel emits the collection entries as individual items.
 :::
 
 Thus the following two declarations produce an identical result even though in the first case the items are specified as multiple arguments while in the second case as a single list object argument:
 
 ```groovy
-Channel.from( 1, 3, 5, 7, 9 )
-Channel.from( [1, 3, 5, 7, 9] )
+channel.from( 1, 3, 5, 7, 9 )
+channel.from( [1, 3, 5, 7, 9] )
 ```
 
 But when more than one argument is provided, they are always managed as *single* emissions. Thus, the following example creates a channel emitting three entries each of which is a list containing two elements:
 
 ```groovy
-Channel.from( [1, 2], [5,6], [7,9] )
+channel.from( [1, 2], [5,6], [7,9] )
 ```
 
 (channel-fromlist)=
@@ -129,10 +135,10 @@ Channel.from( [1, 2], [5,6], [7,9] )
 :::{versionadded} 19.10.0
 :::
 
-The `fromList` method allows you to create a channel emitting the values provided as a list of elements, for example:
+The `channel.fromList` method allows you to create a channel emitting the values provided as a list of elements, for example:
 
 ```groovy
-Channel
+channel
     .fromList( ['a', 'b', 'c', 'd'] )
     .view { "value: $it" }
 ```
@@ -146,28 +152,31 @@ value: c
 value: d
 ```
 
-See also: [of](#of) factory method.
+See also: [channel.of](#of) factory method.
 
 (channel-path)=
 
 ### fromPath
 
-You can create a channel emitting one or more file paths by using the `fromPath` method and specifying a path string as an argument. For example:
+You can create a channel emitting one or more file paths by using the `channel.fromPath` method and specifying a path
+string as an argument. For example:
 
 ```groovy
-myFileChannel = Channel.fromPath( '/data/some/bigfile.txt' )
+myFileChannel = channel.fromPath( '/data/some/bigfile.txt' )
 ```
 
-The above line creates a channel and binds it to a [Path](http://docs.oracle.com/javase/7/docs/api/java/nio/file/Path.html) object for the specified file.
+The above line creates a channel and binds it to a [Path](http://docs.oracle.com/javase/7/docs/api/java/nio/file/Path.html)
+object for the specified file.
 
 :::{note}
-`fromPath` does not check whether the file exists.
+`channel.fromPath` does not check whether the file exists.
 :::
 
-Whenever the `fromPath` argument contains a `*` or `?` wildcard character it is interpreted as a [glob][glob] path matcher. For example:
+Whenever the `channel.fromPath` argument contains a `*` or `?` wildcard character it is interpreted as a [glob][glob] path matcher.
+For example:
 
 ```groovy
-myFileChannel = Channel.fromPath( '/data/big/*.txt' )
+myFileChannel = channel.fromPath( '/data/big/*.txt' )
 ```
 
 This example creates a channel and emits as many `Path` items as there are files with `txt` extension in the `/data/big` folder.
@@ -179,9 +188,9 @@ Two asterisks, i.e. `**`, works like `*` but crosses directory boundaries. This
 For example:
 
 ```groovy
-files = Channel.fromPath( 'data/**.fa' )
-moreFiles = Channel.fromPath( 'data/**/*.fa' )
-pairFiles = Channel.fromPath( 'data/file_{1,2}.fq' )
+files = channel.fromPath( 'data/**.fa' )
+moreFiles = channel.fromPath( 'data/**/*.fa' )
+pairFiles = channel.fromPath( 'data/file_{1,2}.fq' )
 ```
 
 The first line returns a channel emitting the files ending with the suffix `.fa` in the `data` folder *and* recursively in all its sub-folders. While the second one only emits the files which have the same suffix in *any* sub-folder in the `data` path. Finally the last example emits two files: `data/file_1.fq` and `data/file_2.fq`.
@@ -193,15 +202,15 @@ As in Linux Bash, the `*` wildcard does not catch hidden files (i.e. files whose
 Multiple paths or glob patterns can be specified using a list:
 
 ```groovy
-Channel.fromPath( ['/some/path/*.fq', '/other/path/*.fastq'] )
+channel.fromPath( ['/some/path/*.fq', '/other/path/*.fastq'] )
 ```
 
 In order to include hidden files, you need to start your pattern with a period character or specify the `hidden: true` option. For example:
 
 ```groovy
-expl1 = Channel.fromPath( '/path/.*' )
-expl2 = Channel.fromPath( '/path/.*.fa' )
-expl3 = Channel.fromPath( '/path/*', hidden: true )
+expl1 = channel.fromPath( '/path/.*' )
+expl2 = channel.fromPath( '/path/.*.fa' )
+expl3 = channel.fromPath( '/path/*', hidden: true )
 ```
 
 The first example returns all hidden files in the specified path. The second one returns all hidden files ending with the `.fa` suffix. Finally the last example returns all files (hidden and non-hidden) in that path.
@@ -211,8 +220,8 @@ By default a [glob][glob] pattern only looks for regular file paths that match t
 You can use the `type` option specifying the value `file`, `dir` or `any` in order to define what kind of paths you want. For example:
 
 ```groovy
-myFileChannel = Channel.fromPath( '/path/*b', type: 'dir' )
-myFileChannel = Channel.fromPath( '/path/a*', type: 'any' )
+myFileChannel = channel.fromPath( '/path/*b', type: 'dir' )
+myFileChannel = channel.fromPath( '/path/a*', type: 'any' )
 ```
 
 The first example will return all *directory* paths ending with the `b` suffix, while the second will return any file or directory starting with a `a` prefix.
@@ -244,10 +253,11 @@ Available options:
 
 ### fromFilePairs
 
-The `fromFilePairs` method creates a channel emitting the file pairs matching a [glob][glob] pattern provided by the user. The matching files are emitted as tuples in which the first element is the grouping key of the matching pair and the second element is the list of files (sorted in lexicographical order). For example:
+The `channel.fromFilePairs` method creates a channel emitting the file pairs matching a [glob][glob] pattern provided
+by the user. The matching files are emitted as tuples in which the first element is the grouping key of the matching pair and the second element is the list of files (sorted in lexicographical order). For example:
 
 ```groovy
-Channel
+channel
     .fromFilePairs('/my/data/SRR*_{1,2}.fastq')
     .view()
 ```
@@ -270,13 +280,13 @@ The glob pattern must contain at least one `*` wildcard character.
 Multiple glob patterns can be specified using a list:
 
 ```groovy
-Channel.fromFilePairs( ['/some/data/SRR*_{1,2}.fastq', '/other/data/QFF*_{1,2}.fastq'] )
+channel.fromFilePairs( ['/some/data/SRR*_{1,2}.fastq', '/other/data/QFF*_{1,2}.fastq'] )
 ```
 
 Alternatively, it is possible to implement a custom file pair grouping strategy providing a closure which, given the current file as parameter, returns the grouping key. For example:
 
 ```groovy
-Channel
+channel
     .fromFilePairs('/some/data/*', size: -1) { file -> file.extension }
     .view { ext, files -> "Files with the extension $ext are $files" }
 ```
@@ -311,10 +321,10 @@ Available options:
 :::{versionadded} 19.04.0
 :::
 
-The `fromSRA` method queries the [NCBI SRA](https://www.ncbi.nlm.nih.gov/sra) database and returns a channel emitting the FASTQ files matching the specified criteria i.e project or accession number(s). For example:
+The `channel.fromSRA` method queries the [NCBI SRA](https://www.ncbi.nlm.nih.gov/sra) database and returns a channel emitting the FASTQ files matching the specified criteria i.e project or accession number(s). For example:
 
 ```groovy
-Channel
+channel
     .fromSRA('SRP043510')
     .view()
 ```
@@ -335,7 +345,7 @@ Multiple accession IDs can be specified using a list object:
 
 ```groovy
 ids = ['ERR908507', 'ERR908506', 'ERR908505']
-Channel
+channel
     .fromSRA(ids)
     .view()
 ```
@@ -356,7 +366,7 @@ To access the ESearch API, you must provide your [NCBI API keys](https://ncbiins
 
 - The `apiKey` option:
   ```groovy
-  Channel.fromSRA(ids, apiKey:'0123456789abcdef')
+  channel.fromSRA(ids, apiKey:'0123456789abcdef')
   ```
 
 - The `NCBI_API_KEY` variable in your environment:
@@ -385,14 +395,15 @@ Available options:
 :::{versionadded} 19.10.0
 :::
 
-The `of` method allows you to create a channel that emits the arguments provided to it, for example:
+The `channel.of` method allows you to create a channel that emits the arguments provided to it, for example:
 
 ```groovy
-ch = Channel.of( 1, 3, 5, 7 )
+ch = channel.of( 1, 3, 5, 7 )
 ch.view { "value: $it" }
 ```
 
-The first line in this example creates a variable `ch` which holds a channel object. This channel emits the arguments supplied to the `of` method. Thus the second line prints the following:
+The first line in this example creates a variable `ch` which holds a channel object. This channel emits the arguments
+supplied to the `of` method. Thus the second line prints the following:
 
 ```
 value: 1
@@ -404,7 +415,7 @@ value: 7
 Ranges of values are expanded accordingly:
 
 ```groovy
-Channel
+channel
     .of(1..23, 'X', 'Y')
     .view()
 ```
@@ -422,37 +433,144 @@ X
 Y
 ```
 
-See also: [fromList](#fromlist) factory method.
+See also: [channel.fromList](#fromlist) factory method.
+
+(channel-topic)=
+
+### topic
+
+:::{versionadded} 23.11.0-edge
+:::
+
+:::{note}
+This feature requires the `nextflow.preview.topic` feature flag to be enabled.
+:::
+
+A *topic* is a channel type introduced as of Nextflow 23.11.0-edge along with {ref}`channel-type-value` and
+{ref}`channel-type-queue`.
+
+A *topic channel*, similarly to a *queue channel*, is non-blocking unidirectional FIFO queue, however it connects
+multiple *producer* processes with multiple *consumer* processes or operators.
+
+:::{tip}
+You can think about it as a channel that is shared across many different process using the same *topic name*.
+:::
+
+A process output can be assigned to a topic using the `topic` option on an output, for example:
+
+```groovy
+process foo {
+  output:
+  val('foo'), topic: my_topic
+}
+
+process bar {
+  output:
+  val('bar'), topic: my_topic
+}
+```
+
+The `channel.topic` method allows referencing the topic channel with the specified name, which can be used as a process
+input or operator composition as any other Nextflow channel:
+
+```groovy
+channel.topic('my-topic').view()
+```
+
+This approach is a convenient way to collect related items from many different sources without explicitly defining
+the logic connecting many different queue channels altogether, commonly using the `mix` operator.
+
+:::{warning}
+Any process that consumes a channel topic should not send any outputs to that topic, or else the pipeline will hang forever.
+:::
+
+See also: {ref}`process-additional-options` for process outputs.
+
+(channel-topic)=
+
+### topic
+
+:::{versionadded} 23.11.0-edge
+:::
+
+:::{note}
+This feature requires the `nextflow.preview.topic` feature flag to be enabled.
+:::
+
+A *topic* is a channel type introduced as of Nextflow 23.11.0-edge along with {ref}`channel-type-value` and
+{ref}`channel-type-queue`.
+
+A *topic channel*, similarly to a *queue channel*, is non-blocking unidirectional FIFO queue, however it connects
+multiple *producer* processes with multiple *consumer* processes or operators.
+
+:::{tip}
+You can think about it as a channel that is shared across many different process using the same *topic name*.
+:::
+
+A process output can be assigned to a topic using the `topic` option on an output, for example:
+
+```groovy
+process foo {
+  output:
+  val('foo'), topic: my_topic
+}
+
+process bar {
+  output:
+  val('bar'), topic: my_topic
+}
+```
+
+The `channel.topic` method allows referencing the topic channel with the specified name, which can be used as a process
+input or operator composition as any other Nextflow channel:
+
+```groovy
+Channel.topic('my-topic').view()
+```
+
+This approach is a convenient way to collect related items from many different sources without explicitly defining
+the logic connecting many different queue channels altogether, commonly using the `mix` operator.
+
+:::{warning}
+Any process that consumes a channel topic should not send any outputs to that topic, or else the pipeline will hang forever.
+:::
+
+See also: {ref}`process-additional-options` for process outputs.
 
 (channel-value)=
 
 ### value
 
-The `value` method is used to create a value channel. An optional (not `null`) argument can be specified to bind the channel to a specific value. For example:
+The `channel.value` method is used to create a value channel. An optional (not `null`) argument can be specified to bind
+the channel to a specific value. For example:
 
 ```groovy
-expl1 = Channel.value()
-expl2 = Channel.value( 'Hello there' )
-expl3 = Channel.value( [1,2,3,4,5] )
+expl1 = channel.value()
+expl2 = channel.value( 'Hello there' )
+expl3 = channel.value( [1,2,3,4,5] )
 ```
 
-The first line in the example creates an 'empty' variable. The second line creates a channel and binds a string to it. The third line creates a channel and binds a list object to it that will be emitted as a single value.
+The first line in the example creates an 'empty' variable. The second line creates a channel and binds a string to it.
+The third line creates a channel and binds a list object to it that will be emitted as a single value.
 
 (channel-watchpath)=
 
 ### watchPath
 
-The `watchPath` method watches a folder for one or more files matching a specified pattern. As soon as there is a file that meets the specified condition, it is emitted over the channel that is returned by the `watchPath` method. The condition on files to watch can be specified by using `*` or `?` wildcard characters i.e. by specifying a [glob][glob] path matching criteria.
+The `channel.watchPath` method watches a folder for one or more files matching a specified pattern. As soon as there
+is a file that meets the specified condition, it is emitted over the channel that is returned by the `watchPath` method.
+The condition on files to watch can be specified by using `*` or `?` wildcard characters i.e. by specifying a [glob][glob] path matching criteria.
 
 For example:
 
 ```groovy
-Channel
+channel
     .watchPath( '/path/*.fa' )
     .subscribe { println "Fasta file: $it" }
 ```
 
-By default it watches only for new files created in the specified folder. Optionally, it is possible to provide a second argument that specifies what event(s) to watch. The supported events are:
+By default it watches only for new files created in the specified folder. Optionally, it is possible to provide a second
+argument that specifies what event(s) to watch. The supported events are:
 
 - `create`: A new file is created (default)
 - `modify`: A file is modified
@@ -461,15 +579,17 @@ By default it watches only for new files created in the specified folder. Option
 You can specify more than one of these events by using a comma separated string as shown below:
 
 ```groovy
-Channel
+channel
     .watchPath( '/path/*.fa', 'create,modify' )
     .subscribe { println "File created or modified: $it" }
 ```
 
 :::{warning}
-The `watchPath` factory waits endlessly for files that match the specified pattern and event(s), which means that it will cause your pipeline to run forever. Consider using the `take` or `until` operator to close the channel when a certain condition is met (e.g. after receiving 10 files, receiving a file named `DONE`).
+The `channel.watchPath` factory waits endlessly for files that match the specified pattern and event(s), which means
+that it will cause your pipeline to run forever. Consider using the `take` or `until` operator to close the channel when
+a certain condition is met (e.g. after receiving 10 files, receiving a file named `DONE`).
 :::
 
-See also: [fromPath](#frompath) factory method.
+See also: [channel.fromPath](#frompath) factory method.
 
 [glob]: http://docs.oracle.com/javase/tutorial/essential/io/fileOps.html#glob
diff --git a/docs/conf.py b/docs/conf.py
index 7a9f214174..80b1ac70b2 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -64,9 +64,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '23.10'
+version = '23.11'
 # The full version, including alpha/beta/rc tags.
-release = '23.10.0'
+release = '23.11.0-edge'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/docs/config.md b/docs/config.md
index 954bb43a28..20acf2a51c 100644
--- a/docs/config.md
+++ b/docs/config.md
@@ -169,8 +169,13 @@ The following settings are available:
 `aws.batch.delayBetweenAttempts`
 : Delay between download attempts from S3 (default: `10 sec`).
 
+`aws.batch.executionRole`
+: :::{versionadded} 23.12.0-edge
+  :::
+: The AWS Batch Execution Role ARN that needs to be used to execute the Batch Job. This is mandatory when using AWS Fargate platform type. See [AWS documentation](https://docs.aws.amazon.com/batch/latest/userguide/execution-IAM-role.html) for more details.
+
 `aws.batch.jobRole`
-: The AWS Job Role ARN that needs to be used to execute the Batch Job.
+: The AWS Batch Job Role ARN that needs to be used to execute the Batch Job.
 
 `aws.batch.logsGroup`
 : :::{versionadded} 22.09.0-edge
@@ -188,6 +193,11 @@ The following settings are available:
 `aws.batch.maxTransferAttempts`
 : Max number of downloads attempts from S3 (default: `1`).
 
+`aws.batch.platformType`
+: :::{versionadded} 23.12.0-edge
+  :::
+: Allow specifying the compute platform type used by AWS Batch, that can be either `ec2` or `fargate`. See AWS documentation to learn more about [AWS Fargate platform type](https://docs.aws.amazon.com/batch/latest/userguide/fargate.html) for AWS Batch.
+
 `aws.batch.retryMode`
 : The retry mode configuration setting, to accommodate rate-limiting on [AWS services](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-retries.html) (default: `standard`, other options: `legacy`, `adaptive`); this handling is delegated to AWS. To have Nextflow handle retries instead, use `built-in`.
 
@@ -220,7 +230,10 @@ The following settings are available:
 : :::{versionadded} 22.12.0-edge
   :::
 : *Experimental: may change in a future release.*
-: Enable auto retrieval of S3 objects stored with Glacier class store (default: `false`).
+: Enable auto retrieval of S3 objects with a Glacier storage class (default: `false`).
+: :::{note}
+  This feature only works for S3 objects that are downloaded by Nextflow directly. It is not supported for tasks (e.g. when using the AWS Batch executor), since that would lead to many tasks sitting idle for several hours and wasting resources. If you need to restore many objects from Glacier, consider restoring them in a script prior to launching the pipeline.
+  :::
 
 `aws.client.glacierExpirationDays`
 : :::{versionadded} 22.12.0-edge
@@ -276,7 +289,7 @@ The following settings are available:
 `aws.client.storageKmsKeyId`
 : :::{versionadded} 22.05.0-edge
   :::
-: The AWS KMS key Id to be used to encrypt files stored in the target S3 bucket ().
+: The AWS KMS key Id to be used to encrypt files stored in the target S3 bucket.
 
 `aws.client.userAgent`
 : The HTTP user agent header passed with all HTTP requests.
@@ -363,6 +376,10 @@ The following settings are available:
 : *New in `nf-azure` version `0.11.0`*
 : If mounting File Shares, this is the internal root mounting point. Must be `/mnt/resource/batch/tasks/fsmounts` for CentOS nodes or `/mnt/batch/tasks/fsmounts` for Ubuntu nodes (default is for CentOS).
 
+`azure.batch.pools..lowPriority`
+: *New in `nf-azure` version `1.4.0`*
+: Enable the use of low-priority VMs (default: `false`).
+
 `azure.batch.pools..maxVmCount`
 : Specify the max of virtual machine when using auto scale option.
 
@@ -748,7 +765,9 @@ The `google` scope allows you to configure the interactions with Google Cloud, i
 
 Read the {ref}`google-page` page for more information.
 
-The following settings are available:
+#### Cloud Batch
+
+The following settings are available for Google Cloud Batch:
 
 `google.enableRequesterPaysBuckets`
 : When `true` uses the given Google Cloud project ID as the billing project for storage access. This is required when accessing data from *requester pays enabled* buckets. See [Requester Pays on Google Cloud Storage documentation](https://cloud.google.com/storage/docs/requester-pays) (default: `false`).
@@ -766,17 +785,14 @@ The following settings are available:
 `google.location`
 : The Google Cloud location where jobs are executed (default: `us-central1`).
 
+`google.batch.maxSpotAttempts`
+: :::{versionadded} 23.11.0-edge
+  :::
+: Max number of execution attempts of a job interrupted by a Compute Engine spot reclaim event (default: `5`).
+
 `google.project`
 : The Google Cloud project ID to use for pipeline execution
 
-`google.region`
-: *Available only for Google Life Sciences*
-: The Google Cloud region where jobs are executed. Multiple regions can be provided as a comma-separated list. Cannot be used with the `google.zone` option. See the [Google Cloud documentation](https://cloud.google.com/compute/docs/regions-zones/) for a list of available regions and zones.
-
-`google.zone`
-: *Available only for Google Life Sciences*
-: The Google Cloud zone where jobs are executed. Multiple zones can be provided as a comma-separated list. Cannot be used with the `google.region` option. See the [Google Cloud documentation](https://cloud.google.com/compute/docs/regions-zones/) for a list of available regions and zones.
-
 `google.batch.allowedLocations`
 : :::{versionadded} 22.12.0-edge
   :::
@@ -815,6 +831,50 @@ The following settings are available:
 `google.batch.usePrivateAddress`
 : When `true` the VM will NOT be provided with a public IP address, and only contain an internal IP. If this option is enabled, the associated job can only load docker images from Google Container Registry, and the job executable cannot use external services other than Google APIs (default: `false`).
 
+`google.storage.maxAttempts`
+: :::{versionadded} 23.11.0-edge
+  :::
+: Max attempts when retrying failed API requests to Cloud Storage (default: `10`).
+
+`google.storage.maxDelay`
+: :::{versionadded} 23.11.0-edge
+  :::
+: Max delay when retrying failed API requests to Cloud Storage (default: `'90s'`).
+
+`google.storage.multiplier`
+: :::{versionadded} 23.11.0-edge
+  :::
+: Delay multiplier when retrying failed API requests to Cloud Storage (default: `2.0`).
+
+#### Cloud Life Sciences
+
+The following settings are available for Cloud Life Sciences:
+
+`google.enableRequesterPaysBuckets`
+: When `true` uses the given Google Cloud project ID as the billing project for storage access. This is required when accessing data from *requester pays enabled* buckets. See [Requester Pays on Google Cloud Storage documentation](https://cloud.google.com/storage/docs/requester-pays) (default: `false`).
+
+`google.httpConnectTimeout`
+: :::{versionadded} 23.06.0-edge
+  :::
+: Defines the HTTP connection timeout for Cloud Storage API requests (default: `'60s'`).
+
+`google.httpReadTimeout`
+: :::{versionadded} 23.06.0-edge
+  :::
+: Defines the HTTP read timeout for Cloud Storage API requests (default: `'60s'`).
+
+`google.location`
+: The Google Cloud location where jobs are executed (default: `us-central1`).
+
+`google.project`
+: The Google Cloud project ID to use for pipeline execution
+
+`google.region`
+: The Google Cloud region where jobs are executed. Multiple regions can be provided as a comma-separated list. Cannot be used with the `google.zone` option. See the [Google Cloud documentation](https://cloud.google.com/compute/docs/regions-zones/) for a list of available regions and zones.
+
+`google.zone`
+: The Google Cloud zone where jobs are executed. Multiple zones can be provided as a comma-separated list. Cannot be used with the `google.region` option. See the [Google Cloud documentation](https://cloud.google.com/compute/docs/regions-zones/) for a list of available regions and zones.
+
 `google.lifeSciences.bootDiskSize`
 : Set the size of the virtual machine boot disk e.g `50.GB` (default: none).
 
@@ -1089,8 +1149,8 @@ Read the {ref}`sharing-page` page to learn how to publish your pipeline to GitHu
 
 The `notification` scope allows you to define the automatic sending of a notification email message when the workflow execution terminates.
 
-`notification.binding`
-: An associative array modelling the variables in the template file.
+`notification.attributes`
+: A map object modelling the variables that can be used in the template file.
 
 `notification.enabled`
 : Enables the sending of a notification message when the workflow execution completes.
@@ -1350,7 +1410,7 @@ The following settings are available:
 `singularity.oci`
 : :::{versionadded} 23.11.0-edge
   :::
-: Enable OCI-mode the allows the use of native OCI-compatible containers with Singularity. See [Singularity documentation](https://docs.sylabs.io/guides/4.0/user-guide/oci_runtime.html#oci-mode) for more details and requirements (default: `false`).
+: Enable OCI-mode, that allows running native OCI-compatible containers with Singularity using `crun` or `runc` as low-level runtime. See `--oci` flag in the [Singularity documentation](https://docs.sylabs.io/guides/4.0/user-guide/oci_runtime.html#oci-mode) for more details and requirements (default: `false`).
 
 
 `singularity.pullTimeout`
@@ -1806,3 +1866,12 @@ Some features can be enabled using the `nextflow.enable` and `nextflow.preview`
 : *Experimental: may change in a future release.*
 
 : When `true`, enables process and workflow recursion. See [this GitHub discussion](https://github.com/nextflow-io/nextflow/discussions/2521) for more information.
+
+`nextflow.preview.topic`
+
+: :::{versionadded} 23.11.0-edge
+  :::
+
+: *Experimental: may change in a future release.*
+
+: When `true`, enables {ref}`topic channels ` feature.
diff --git a/docs/container.md b/docs/container.md
index a1f3799693..cb5d50547d 100644
--- a/docs/container.md
+++ b/docs/container.md
@@ -5,7 +5,7 @@
 Nextflow supports a variety of container runtimes. Containerization allows you to write self-contained and truly reproducible computational pipelines, by packaging the binary dependencies of a script into a standard and portable format that can be executed on any platform that supports a container runtime. Furthermore, the same pipeline can be transparently executed with any of the supported container runtimes, depending on which runtimes are available in the target compute environment.
 
 :::{note}
-When creating your container image to use with Nextflow, make sure that Bash (3.x or later) and `ps` are installed in your image, along with other tools required for collecting metrics (See  {ref}`this section `). Also, Bash should be available on the path `/bin/bash` and it should be the container entrypoint.
+When creating a container image to use with Nextflow, make sure that Bash (3.x or later) and `ps` are installed in the image, along with other tools required for collecting metrics (See {ref}`this section `). Bash should be available on the path `/bin/bash` and it should be the container entrypoint.
 :::
 
 (container-apptainer)=
diff --git a/docs/executor.md b/docs/executor.md
index 0f50eff6ae..08aba7bc2d 100644
--- a/docs/executor.md
+++ b/docs/executor.md
@@ -23,9 +23,11 @@ The pipeline can be launched either in a local computer, or an EC2 instance. EC2
 Resource requests and other job characteristics can be controlled via the following process directives:
 
 - {ref}`process-accelerator`
+- {ref}`process-arch` (only when using Fargate platform type for AWS Batch)
 - {ref}`process-container`
 - {ref}`process-containerOptions`
 - {ref}`process-cpus`
+- {ref}`process-disk` (only when using Fargate platform type for AWS Batch)
 - {ref}`process-memory`
 - {ref}`process-queue`
 - {ref}`process-resourcelabels`
diff --git a/docs/fusion.md b/docs/fusion.md
index 186837e76b..04a07e7928 100644
--- a/docs/fusion.md
+++ b/docs/fusion.md
@@ -184,15 +184,20 @@ The following configuration options are available:
 `fusion.enabled`
 : Enable/disable the use of Fusion file system.
 
+`fusion.cacheSize`
+: :::{versionadded} 23.11.0-edge
+:::
+: The maximum size of the local cache used by the Fusion client.
+
+`fusion.containerConfigUrl`
+: The URL from where the container layer provisioning the Fusion client is downloaded.
+
 `fusion.exportStorageCredentials`
 : :::{versionadded} 23.05.0-edge
   This option was previously named `fusion.exportAwsAccessKeys`.
   :::
 : When `true` the access credentials required by the underlying object storage are exported the pipeline jobs execution environment.
 
-`fusion.containerConfigUrl`
-: The URL from where the container layer provisioning the Fusion client is downloaded. 
-
 `fusion.logLevel`
 : The level of logging emitted by the Fusion client.
 
@@ -207,8 +212,6 @@ The following configuration options are available:
   executor which requires the use the [k8s-fuse-plugin](https://github.com/nextflow-io/k8s-fuse-plugin) to be installed
   in the target cluster (default: `true`).
 
-`fusion.tagsEnabled`
-: Enable/disable the tagging of files created in the underlying object storage via the Fusion client (default: `true`).
-
-`fusion.tagsPattern`
-: The pattern that determines how tags are applied to files created via the Fusion client (default: `[.command.*|.exitcode|.fusion.*](nextflow.io/metadata=true),[*](nextflow.io/temporary=true)`)
+`fusion.tags`
+: The pattern that determines how tags are applied to files created via the Fusion client. To disable tags
+  set it to `false`. (default: `[.command.*|.exitcode|.fusion.*](nextflow.io/metadata=true),[*](nextflow.io/temporary=true)`)
diff --git a/docs/mail.md b/docs/mail.md
index 59ce19890a..093b56fb1c 100644
--- a/docs/mail.md
+++ b/docs/mail.md
@@ -10,7 +10,7 @@ The built-in function `sendMail` allows you to send a mail message from a workfl
 
 ### Basic mail
 
-The mail attributes are specified as named parameters or providing an equivalent associative array as argument. For example:
+The mail attributes are specified as named parameters or an equivalent map. For example:
 
 ```groovy
 sendMail(
diff --git a/docs/operator.md b/docs/operator.md
index a54f0b022e..8ba07a7cc7 100644
--- a/docs/operator.md
+++ b/docs/operator.md
@@ -8,7 +8,7 @@ This page is a comprehensive reference for all Nextflow operators. However, if y
 
 - Filtering: [filter](#filter), [randomSample](#randomsample), [take](#take), [unique](#unique)
 - Reduction: [collect](#collect), [groupTuple](#grouptuple), [reduce](#reduce)
-- Parsing text data: [splitCsv](#splitcsv), [splitJson](#splitjson), [splitText](#splittext)
+- Text processing: [splitCsv](#splitcsv), [splitJson](#splitjson), [splitText](#splittext)
 - Combining channels: [combine](#combine), [concat](#concat), [join](#join), [mix](#mix)
 - Forking channels: [branch](#branch), [multiMap](#multimap)
 - Maths: [count](#count), [max](#max), [min](#min), [sum](#sum)
@@ -23,9 +23,9 @@ This page is a comprehensive reference for all Nextflow operators. However, if y
 
 *Returns: map of queue channels*
 
-The `branch` operator allows you to forward the items emitted by a source channel to one or more output channels, choosing one out of them at a time.
+The `branch` operator forwards each item from a source channel to one of multiple output channels, based on a selection criteria.
 
-The selection criteria is defined by specifying a {ref}`closure ` that provides one or more boolean expression, each of which is identified by a unique label. On the first expression that evaluates to a *true* value, the current item is bound to a named channel as the label identifier. For example:
+The selection criteria is a {ref}`closure ` that defines, for each output channel, a unique label followed by a boolean expression. When an item is received, it is routed to the first output channel whose expression evaluates to `true`. For example:
 
 ```{literalinclude} snippets/branch.nf
 :language: groovy
@@ -36,10 +36,10 @@ The selection criteria is defined by specifying a {ref}`closure 
 ```
 
 :::{note}
-The above *small* and *large* strings may be printed in any order due to the asynchronous execution of the `view` operator.
+The above output may be printed in any order since the two `view` operations are executed asynchronously.
 :::
 
-A default fallback condition can be specified using `true` as the last branch condition:
+A fallback condition can be specified using `true` as the last branch condition:
 
 ```{literalinclude} snippets/branch-with-fallback.nf
 :language: groovy
@@ -49,7 +49,7 @@ A default fallback condition can be specified using `true` as the last branch co
 :language: console
 ```
 
-The value returned by each branch condition can be customised by specifying an optional expression statement(s) just after the condition expression. For example:
+The value emitted to each branch can be customized with an expression statement (or statements) after the branch condition:
 
 ```{literalinclude} snippets/branch-with-mapper.nf
 :language: groovy
@@ -63,7 +63,7 @@ The value returned by each branch condition can be customised by specifying an o
 When the `return` keyword is omitted, the value of the last expression statement is implicitly returned.
 :::
 
-To create a branch criteria as variable that can be passed as an argument to more than one `branch` operator use the `branchCriteria` built-in method as shown below:
+The `branchCriteria()` method can be used to create a branch criteria as a variable that can be passed as an argument to any number of `branch` operations, as shown below:
 
 ```{literalinclude} snippets/branch-criteria.nf
 :language: groovy
@@ -77,11 +77,13 @@ To create a branch criteria as variable that can be passed as an argument to mor
 
 *Returns: queue channel*
 
-The `buffer` operator gathers the items emitted by the source channel into subsets and emits these subsets separately.
+The `buffer` operator collects items from a source channel into subsets and emits each subset separately.
 
-There are a number of ways you can regulate how `buffer` gathers the items from the source channel into subsets:
+This operator has multiple variants:
 
-- `buffer( closingCondition )`: starts to collect the items emitted by the channel into a subset until the `closingCondition` is verified. After that the subset is emitted to the resulting channel and new items are gathered into a new subset. The process is repeated until the last value in the source channel is sent. The `closingCondition` can be specified either as a {ref}`regular expression `, a Java class, a literal value, or a boolean predicate that has to be satisfied. For example:
+`buffer( closingCondition )`
+
+: Emits each subset when `closingCondition` is satisfied. The closing condition can be a literal value, a {ref}`regular expression `, a type qualifier (i.e. Java class), or a boolean predicate. For example:
 
   ```{literalinclude} snippets/buffer-with-closing.nf
   :language: groovy
@@ -91,7 +93,9 @@ There are a number of ways you can regulate how `buffer` gathers the items from
   :language: console
   ```
 
-- `buffer( openingCondition, closingCondition )`: starts to gather the items emitted by the channel as soon as one of the them verify the `openingCondition` and it continues until there is one item which verify the `closingCondition`. After that the subset is emitted and it continues applying the described logic until the last channel item is emitted. Both conditions can be defined either as a {ref}`regular expression `, a literal value, a Java class, or a boolean predicate that need to be satisfied. For example:
+`buffer( openingCondition, closingCondition )`
+
+: Creates a new subset when `openingCondition` is satisfied and emits the subset when is `closingCondition` is satisfied. The opening and closing conditions can each be a literal value, a {ref}`regular expression `, a type qualifier (i.e. Java class), or a boolean predicate. For example:
 
   ```{literalinclude} snippets/buffer-with-opening-closing.nf
   :language: groovy
@@ -101,7 +105,9 @@ There are a number of ways you can regulate how `buffer` gathers the items from
   :language: console
   ```
 
-- `buffer( size: n )`: transform the source channel in such a way that it emits tuples made up of `n` elements. An incomplete tuple is discarded. For example:
+`buffer( size: n )`
+
+: Emits a new subset for every `n` items. Remaining items are discarded. For example:
 
   ```{literalinclude} snippets/buffer-with-size.nf
   :language: groovy
@@ -111,7 +117,7 @@ There are a number of ways you can regulate how `buffer` gathers the items from
   :language: console
   ```
 
-  If you want to emit the last items in a tuple containing less than `n` elements, simply add the parameter `remainder` specifying `true`, for example:
+  The `remainder` option can be used to emit any remaining items as a partial subset:
 
   ```{literalinclude} snippets/buffer-with-size-remainder.nf
   :language: groovy
@@ -121,7 +127,9 @@ There are a number of ways you can regulate how `buffer` gathers the items from
   :language: console
   ```
 
-- `buffer( size: n, skip: m )`: as in the previous example, it emits tuples containing `n` elements, but skips `m` values before starting to collect the values for the next tuple (including the first emission). For example:
+`buffer( size: n, skip: m )`
+
+: Emits a new subset for every `n` items, skipping `m` items before collecting each subset. For example:
 
   ```{literalinclude} snippets/buffer-with-size-skip.nf
   :language: groovy
@@ -131,47 +139,59 @@ There are a number of ways you can regulate how `buffer` gathers the items from
   :language: console
   ```
 
-  If you want to emit the remaining items in a tuple containing less than `n` elements, simply add the parameter `remainder` specifying `true`, as shown in the previous example.
+  The `remainder` option can be used to emit any remaining items as a partial subset.
 
-See also: [collate](#collate) operator.
+See also: [collate](#collate)
 
 ## collate
 
 *Returns: queue channel*
 
-The `collate` operator transforms a channel in such a way that the emitted values are grouped in tuples containing `n` items. For example:
+The `collate` operator collects items from a source channel into groups of *N* items.
 
-```{literalinclude} snippets/collate.nf
-:language: groovy
-```
+This operator has multiple variants:
 
-```{literalinclude} snippets/collate.out
-:language: console
-```
+`collate( size, remainder = true )`
 
-As shown in the above example the last tuple may be incomplete e.g. contain fewer elements than the specified size. If you want to avoid this, specify `false` as the second parameter. For example:
+: Collects items into groups of `size` items:
 
-```{literalinclude} snippets/collate-with-no-remainder.nf
-:language: groovy
-```
+  ```{literalinclude} snippets/collate.nf
+  :language: groovy
+  ```
 
-```{literalinclude} snippets/collate-with-no-remainder.out
-:language: console
-```
+  ```{literalinclude} snippets/collate.out
+  :language: console
+  ```
 
-A second version of the `collate` operator allows you to specify, after the `size`, the `step` by which elements are collected in tuples. For example:
+  By default, any remaining items are emitted as a partial group. You can specify `false` as the second parameter to discard them instead:
 
-```{literalinclude} snippets/collate-with-step.nf
-:language: groovy
-```
+  ```{literalinclude} snippets/collate-with-no-remainder.nf
+  :language: groovy
+  ```
 
-```{literalinclude} snippets/collate-with-step.out
-:language: console
-```
+  ```{literalinclude} snippets/collate-with-no-remainder.out
+  :language: console
+  ```
 
-As before, if you don't want to emit the last items which do not complete a tuple, specify `false` as the third parameter.
+  :::{note}
+  This version of `collate` is equivalent to `buffer( size: n, remainder: true | false )`.
+  :::
+
+`collate( size, step, remainder = true )`
+
+: Collects items into groups of `size` items using a *sliding window* that moves by `step` items at a time:
+
+  ```{literalinclude} snippets/collate-with-step.nf
+  :language: groovy
+  ```
 
-See also: [buffer](#buffer) operator.
+  ```{literalinclude} snippets/collate-with-step.out
+  :language: console
+  ```
+
+  You can specify `false` as the third parameter to discard any remaining items.
+
+See also: [buffer](#buffer)
 
 (operator-collect)=
 
@@ -179,7 +199,7 @@ See also: [buffer](#buffer) operator.
 
 *Returns: value channel*
 
-The `collect` operator collects all the items emitted by a channel to a `List` and return the resulting object as a sole emission. For example:
+The `collect` operator collects all items from a source channel into a list and emits it as a single item:
 
 ```{literalinclude} snippets/collect.nf
 :language: groovy
@@ -189,7 +209,7 @@ The `collect` operator collects all the items emitted by a channel to a `List` a
 :language: console
 ```
 
-An optional {ref}`closure ` can be specified to transform each item before adding it to the resulting list. For example:
+An optional {ref}`closure ` can be used to transform each item before it is collected:
 
 ```{literalinclude} snippets/collect-with-mapper.nf
 :language: groovy
@@ -202,39 +222,57 @@ An optional {ref}`closure ` can be specified to transform each i
 Available options:
 
 `flat`
-: When `true` nested list structures are normalised and their items are added to the resulting list object (default: `true`).
+: When `true`, nested list structures are flattened and their items are collected individually (default: `true`).
 
 `sort`
-: When `true` the items in the resulting list are sorted by their natural ordering. It is possible to provide a custom ordering criteria by using either a {ref}`closure ` or a [Comparator](https://docs.oracle.com/javase/8/docs/api/java/util/Comparator.html) object (default: `false`).
+: When `true`, the collected items are sorted by their natural ordering (default: `false`). Can also be a {ref}`closure ` or a [Comparator](https://docs.oracle.com/javase/8/docs/api/java/util/Comparator.html) which defines how items are compared during sorting.
 
-See also: [toList](#tolist) and [toSortedList](#tosortedlist) operator.
+See also: [toList](#tolist), [toSortedList](#tosortedlist)
 
 ## collectFile
 
 *Returns: queue channel*
 
-The `collectFile` operator allows you to gather the items emitted by a channel and save them to one or more files. The operator returns a new channel that emits the collected file(s).
+The `collectFile` operator collects the items from a source channel and saves them to one or more files, emitting the collected file(s).
 
-In the simplest case, just specify the name of a file where the entries have to be stored. For example:
+This operator has multiple variants:
 
-```{literalinclude} snippets/collectfile.nf
-:language: groovy
-```
+`collectFile( name: '...', options = [:] )`
 
+: Collects the items and saves them to a single file specified by the `name` option:
 
-A second version of the `collectFile` operator allows you to gather the items emitted by a channel and group them together into files whose name can be defined by a dynamic criteria. The grouping criteria is specified by a {ref}`closure ` that must return a pair in which the first element defines the file name for the group and the second element the actual value to be appended to that file. For example:
+  ```{literalinclude} snippets/collectfile.nf
+  :language: groovy
+  ```
 
-```{literalinclude} snippets/collectfile-closure.nf
-:language: groovy
-```
+`collectFile( closure, options = [:] )`
 
-```{literalinclude} snippets/collectfile-closure.out
-:language: console
-```
+: Collects the items into groups and saves each group to a file, using a grouping criteria. The grouping criteria is a {ref}`closure ` that maps each item to a pair, where the first element is the file name for the group and the second element is the content to be appended to that file. For example:
 
+  ```{literalinclude} snippets/collectfile-closure.nf
+  :language: groovy
+  ```
 
-:::{tip}
-When the items emitted by the source channel are files, the grouping criteria can be omitted. In this case the items content will be grouped into file(s) having the same name as the source items.
+  ```{literalinclude} snippets/collectfile-closure.out
+  :language: console
+  ```
+
+  When the items from the source channel are files, the grouping criteria can be omitted. In this case, the items will be grouped by their source filename.
+
+The following example shows how to use a closure to collect and sort all sequences in a FASTA file from shortest to longest:
+
+```groovy
+Channel
+    .fromPath('/data/sequences.fa')
+    .splitFasta( record: [id: true, sequence: true] )
+    .collectFile( name: 'result.fa', sort: { it.size() } ) {
+        it.sequence
+    }
+    .view { it.text }
+```
+
+:::{warning}
+The `collectFile` operator needs to store files in a temporary directory that is automatically deleted on workflow completion. For performance reasons, this directory is located in the machine's local storage, and it should have as much free space as the data that is being collected. The `tempDir` option can be used to specify a different temporary directory.
 :::
 
 Available options:
@@ -243,7 +281,7 @@ Available options:
 : Controls the caching ability of the `collectFile` operator when using the *resume* feature. It follows the same semantic of the {ref}`process-cache` directive (default: `true`).
 
 `keepHeader`
-: Prepend the resulting file with the header fetched in the first collected file. The header size (ie. lines) can be specified by using the `skip` parameter (default: `false`), to determine how many lines to remove from all collected files except for the first (where no lines will be removed).
+: Prepend the resulting file with the header fetched in the first collected file. The header size (ie. lines) can be specified by using the `skip` option (default: `0`), to determine how many lines to remove from all collected files except for the first (where no lines will be removed).
 
 `name`
 : Name of the file where all received values are stored.
@@ -252,10 +290,10 @@ Available options:
 : Appends a `newline` character automatically after each entry (default: `false`).
 
 `seed`
-: A value or a map of values used to initialise the files content.
+: A value or a map of values used to initialize the files content.
 
 `skip`
-: Skip the first `n` lines e.g. `skip: 1`.
+: Skip the first `n` lines e.g. `skip: 1` (default: `0`).
 
 `sort`
 : Defines sorting criteria of content in resulting file(s). Can be one of the following values:
@@ -265,56 +303,25 @@ Available options:
   - `'index'`: Order the content by the incremental index number assigned to each entry while they are collected.
   - `'hash'`: (default) Order the content by the hash number associated to each entry
   - `'deep'`: Similar to the previous, but the hash number is created on actual entries content e.g. when the entry is a file the hash is created on the actual file content.
-  - A custom sorting criteria can be specified by using either a {ref}`Closure ` or a [Comparator](http://docs.oracle.com/javase/7/docs/api/java/util/Comparator.html) object.
+  - A custom sorting criteria can be specified with a {ref}`Closure ` or a [Comparator](http://docs.oracle.com/javase/7/docs/api/java/util/Comparator.html) object.
 
   The file content is sorted in such a way that it does not depend on the order in which entries were added to it, which guarantees that it is consistent (i.e. does not change) across different executions with the same data.
 
 `storeDir`
-: Folder where the resulting file(s) are be stored.
+: Folder where the resulting file(s) are stored.
 
 `tempDir`
 : Folder where temporary files, used by the collecting process, are stored.
 
-The following snippet shows how sort the content of the result file alphabetically:
-
-```groovy
-Channel
-    .of('Z'..'A')
-    .collectFile(name:'result', sort: true, newLine: true)
-    .view { it.text }
-```
-
-```
-A
-B
-C
-:
-Z
-```
-
-The following example shows how use a `closure` to collect and sort all sequences in a FASTA file from shortest to longest:
-
-```groovy
-Channel
-    .fromPath('/data/sequences.fa')
-    .splitFasta( record: [id: true, sequence: true] )
-    .collectFile( name:'result.fa', sort: { it.size() } ) {
-        it.sequence
-    }
-    .view { it.text }
-```
-
-:::{warning}
-The `collectFile` operator needs to store files in a temporary folder that is automatically deleted on workflow completion. For performance reasons this folder is located in the machine's local storage, and it will require as much free space as the data that is being collected. Optionally, a different temporary data folder can be specified by using the `tempDir` parameter.
-:::
-
 (operator-combine)=
 
 ## combine
 
 *Returns: queue channel*
 
-The `combine` operator combines (cartesian product) the items emitted by two channels or by a channel and a `Collection` object (as right operand). For example:
+The `combine` operator produces the combinations (i.e. cross product, "Cartesian" product) of two source channels, or a channel and a list (as the right operand), emitting each combination separately.
+
+For example:
 
 ```{literalinclude} snippets/combine.nf
 :language: groovy
@@ -324,7 +331,7 @@ The `combine` operator combines (cartesian product) the items emitted by two cha
 :language: console
 ```
 
-A second version of the `combine` operator allows you to combine items that share a common matching key. The index of the key element is specified by using the `by` parameter (zero-based index, multiple indices can be specified as a list of integers). For example:
+The `by` option can be used to combine items that share a matching key. The value should be the zero-based index of the tuple, or a list of indices. For example:
 
 ```{literalinclude} snippets/combine-by.nf
 :language: groovy
@@ -334,7 +341,17 @@ A second version of the `combine` operator allows you to combine items that shar
 :language: console
 ```
 
-See also [join](#join) and [cross](#cross).
+:::{note}
+The `combine` operator is similar to `cross` and `join`, making them easy to confuse. Their differences can be summarized as follows:
+
+- `combine` and `cross` both produce an *outer product* or *cross product*, whereas `join` produces an *inner product*.
+
+- `combine` filters pairs with a matching key only if the `by` option is used, whereas `cross` always filters pairs with a matching key.
+
+- `combine` with the `by` option merges and flattens each pair, whereas `cross` does not. Compare the examples for `combine` and `cross` to see this difference.
+:::
+
+See also: [cross](#cross), [join](#join)
 
 (operator-concat)=
 
@@ -342,9 +359,9 @@ See also [join](#join) and [cross](#cross).
 
 *Returns: queue channel*
 
-The `concat` operator allows you to *concatenate* the items emitted by two or more channels to a new channel. The items emitted by the resulting channel are in the same order as specified in the operator arguments.
+The `concat` operator emits the items from two or more source channels into a single output channel. Each source channel is emitted in the order in which it was specified.
 
-In other words, given *N* channels, the items from the *i+1 th* channel are emitted only after all of the items from the *i th* channel have been emitted.
+In other words, given *N* channels, the items from the *i+1*-th channel are emitted only after all of the items from the *i*-th channel have been emitted.
 
 For example:
 
@@ -356,13 +373,15 @@ For example:
 :language: console
 ```
 
+See also: [mix](#mix)
+
 (operator-count)=
 
 ## count
 
 *Returns: value channel*
 
-The `count` operator creates a channel that emits a single item: a number that represents the total number of items emitted by the source channel. For example:
+The `count` operator computes the total number of items in a source channel and emits it:
 
 ```{literalinclude} snippets/count.nf
 :language: groovy
@@ -372,7 +391,7 @@ The `count` operator creates a channel that emits a single item: a number that r
 :language: console
 ```
 
-An optional parameter can be provided to select which items are to be counted. The selection criteria can be specified either as a {ref}`regular expression `, a literal value, a Java class, or a boolean predicate that needs to be satisfied. For example:
+An optional filter can be provided to select which items to count. The selection criteria can be a literal value, a {ref}`regular expression `, a type qualifier (i.e. Java class), or a boolean predicate. For example:
 
 ```{literalinclude} snippets/count-with-filter-number.nf
 :language: groovy
@@ -436,9 +455,9 @@ Counts the total number of lines in a channel of text files, equivalent to `spli
 
 *Returns: queue channel*
 
-The `cross` operator allows you to combine the items of two channels in such a way that the items of the source channel are emitted along with the items emitted by the target channel for which they have a matching key.
+The `cross` operator emits every pairwise combination of two channels for which the pair has a matching key.
 
-The key is defined, by default, as the first entry in an array, a list or map object, or the value itself for any other data type. For example:
+By default, the key is defined as the first entry in a list or map, or the value itself for any other data type. For example:
 
 ```{literalinclude} snippets/cross.nf
 :language: groovy
@@ -448,13 +467,6 @@ The key is defined, by default, as the first entry in an array, a list or map ob
 :language: console
 ```
 
-The above example shows how the items emitted by the source channels are associated to the ones emitted by the target channel (on the right) having the same key.
-
-There are two important caveats when using the `cross` operator:
-
-1. The operator is not `commutative`, i.e. the result of `a.cross(b)` is different from `b.cross(a)`
-2. The source channel should emits items for which there's no key repetition i.e. the emitted items have an unique key identifier.
-
 An optional closure can be used to define the matching key for each item:
 
 ```{literalinclude} snippets/cross-with-mapper.nf
@@ -465,11 +477,18 @@ An optional closure can be used to define the matching key for each item:
 :language: console
 ```
 
+There are two important caveats when using the `cross` operator:
+
+1. The operator is not *commutative*, i.e. `a.cross(b)` is not the same as `b.cross(a)`
+2. Each source channel should not emit any items with duplicate keys, i.e. each item should have a unique key.
+
+See also: [combine](#combine)
+
 ## distinct
 
 *Returns: queue channel*
 
-The `distinct` operator allows you to remove *consecutive* duplicated items from a channel, so that each emitted item is different from the preceding one. For example:
+The `distinct` operator forwards a source channel with *consecutively* repeated items removed, such that each emitted item is different from the preceding one:
 
 ```{literalinclude} snippets/distinct.nf
 :language: groovy
@@ -479,7 +498,7 @@ The `distinct` operator allows you to remove *consecutive* duplicated items from
 :language: console
 ```
 
-You can also specify an optional {ref}`closure ` that customizes the way it distinguishes between distinct items. For example:
+An optional {ref}`closure ` can be used to transform each value before it is evaluated for distinct-ness:
 
 ```{literalinclude} snippets/distinct-with-mapper.nf
 :language: groovy
@@ -489,17 +508,17 @@ You can also specify an optional {ref}`closure ` that customizes
 :language: console
 ```
 
+See also: [unique](#unique)
+
 (operator-dump)=
 
 ## dump
 
 *Returns: queue channel or value channel, depending on the input*
 
-The `dump` operator prints the items emitted by the channel to which is applied only when the option `-dump-channels` is specified on the `run` command line, otherwise it is ignored.
-
-This is useful to enable the debugging of one or more channel content on-demand by using a command line option instead of modifying your script code.
+The `dump` operator prints each item in a source channel when the pipeline is executed with the `-dump-channels` command-line option, otherwise it does nothing. It is a useful way to inspect and debug channels quickly without having to modify the pipeline script.
 
-An optional `tag` parameter allows you to select which channel to dump. For example:
+The `tag` option can be used to select which channels to dump:
 
 ```{literalinclude} snippets/dump.nf
 :language: groovy
@@ -521,9 +540,9 @@ Available options:
 
 *Returns: queue channel*
 
-The `filter` operator allows you to get only the items emitted by a channel that satisfy a condition and discarding all the others. The filtering condition can be specified by using either a {ref}`regular expression `, a literal value, a type qualifier (i.e. a Java class) or any boolean predicate.
+The `filter` operator emits the items from a source channel that satisfy a condition, discarding all other items. The filter condition can be a literal value, a {ref}`regular expression `, a type qualifier (i.e. Java class), or a boolean predicate.
 
-The following example shows how to filter a channel by using a regular expression that returns only strings that begin with `a`:
+The following example filters a channel with a regular expression that only matches strings beginning with `a`:
 
 ```{literalinclude} snippets/filter-regex.nf
 :language: groovy
@@ -533,7 +552,7 @@ The following example shows how to filter a channel by using a regular expressio
 :language: console
 ```
 
-The following example shows how to filter a channel by specifying the type qualifier `Number` so that only numbers are returned:
+The following example filters a channel with the `Number` type qualifier so that only numbers are emitted:
 
 ```{literalinclude} snippets/filter-type.nf
 :language: groovy
@@ -543,7 +562,7 @@ The following example shows how to filter a channel by specifying the type quali
 :language: console
 ```
 
-Finally, a filtering condition can be defined by using any a boolean predicate. A predicate is expressed by a {ref}`closure ` returning a boolean value. For example the following fragment shows how filter a channel emitting numbers so that the odd values are returned:
+The following example filters a channel using a boolean predicate, which is a {ref}`closure ` that returns a boolean value. In this case, the predicate is used to select only odd numbers:
 
 ```{literalinclude} snippets/filter-closure.nf
 :language: groovy
@@ -553,17 +572,13 @@ Finally, a filtering condition can be defined by using any a boolean predicate.
 :language: console
 ```
 
-:::{tip}
-In the above example the filter condition is wrapped in curly brackets, instead of parentheses, because it specifies a {ref}`closure ` as the operator's argument. In reality it is just syntactic sugar for `filter({ it % 2 == 1 })`
-:::
-
 (operator-first)=
 
 ## first
 
 *Returns: value channel*
 
-The `first` operator creates a channel that returns the first item emitted by the source channel, or eventually the first item that matches an optional condition. The condition can be specified by using a {ref}`regular expression`, a Java `class` type or any boolean predicate. For example:
+The `first` operator emits the first item in a source channel, or the first item that matches a condition. The condition can be a {ref}`regular expression`, a type qualifier (i.e. Java class), or a boolean predicate. For example:
 
 ```{literalinclude} snippets/first.nf
 :language: groovy
@@ -603,7 +618,7 @@ When the mapping function returns a map, each key-value pair in the map is emitt
 
 *Returns: queue channel*
 
-The `flatten` operator transforms a channel in such a way that every item of type `Collection` or `Array` is flattened so that each single entry is emitted separately by the resulting channel. For example:
+The `flatten` operator flattens each item from a source channel that is a list or other collection, such that each element in each collection is emitted separately:
 
 ```{literalinclude} snippets/flatten.nf
 :language: groovy
@@ -613,7 +628,9 @@ The `flatten` operator transforms a channel in such a way that every item of typ
 :language: console
 ```
 
-See also: [flatMap](#flatmap) operator.
+As shown in the above example, deeply nested collections are also flattened.
+
+See also: [flatMap](#flatmap)
 
 (operator-grouptuple)=
 
@@ -621,9 +638,9 @@ See also: [flatMap](#flatmap) operator.
 
 *Returns: queue channel*
 
-The `groupTuple` operator collects tuples (or lists) of values emitted by the source channel grouping together the elements that share the same key. Finally it emits a new tuple object for each distinct key collected.
+The `groupTuple` operator collects lists (i.e. *tuples*) from a source channel into groups based on a grouping key. A new tuple is emitted for each distinct key.
 
-In other words, the operator transforms a sequence of tuple like *(K, V, W, ..)* into a new channel emitting a sequence of *(K, list(V), list(W), ..)*
+To be more precise, the operator transforms a sequence of tuples like *(K, V, W, ..)* into a sequence of tuples like *(K, list(V), list(W), ..)*.
 
 For example:
 
@@ -635,7 +652,7 @@ For example:
 :language: console
 ```
 
-By default the first entry in the tuple is used as grouping key. A different key can be chosen by using the `by` parameter and specifying the index of the entry to be used as key (the index is zero-based). For example, grouping by the second value in each tuple:
+By default, the first element of each tuple is used as the grouping key. The `by` option can be used to specify a different index, or list of indices. For example, to group by the second element of each tuple:
 
 ```{literalinclude} snippets/grouptuple-by.nf
 :language: groovy
@@ -645,7 +662,7 @@ By default the first entry in the tuple is used as grouping key. A different key
 :language: console
 ```
 
-By default, if you don't specify a size, the `groupTuple` operator will not emit any groups until *all* inputs have been received. If possible, you should always try to specify the number of expected elements in each group using the `size` option, so that each group can be emitted as soon as it's ready. In cases where the size of each group varies based on the grouping key, you can use the built-in `groupKey` function, which allows you to create a special grouping key with an associated size:
+By default, if you don't specify a size, the `groupTuple` operator will not emit any groups until *all* inputs have been received. If possible, you should always try to specify the number of expected elements in each group using the `size` option, so that each group can be emitted as soon as it's ready. In cases where the size of each group varies based on the grouping key, you can use the built-in `groupKey()` function, which allows you to define a different expected size for each group:
 
 ```{literalinclude} snippets/grouptuple-groupkey.nf
 :language: groovy
@@ -658,22 +675,22 @@ By default, if you don't specify a size, the `groupTuple` operator will not emit
 Available options:
 
 `by`
-: The index (zero based) of the element to be used as grouping key. A key composed by multiple elements can be defined specifying a list of indices e.g. `by: [0,2]`
+: The zero-based index of the element to use as the grouping key. Can also be a list of indices, e.g. `by: [0,2]` (default: `[0]`).
 
 `remainder`
-: When `false` incomplete tuples (i.e. with less than `size` grouped items) are discarded (default). When `true` incomplete tuples are emitted as the ending emission. Only valid when a `size` parameter is specified.
+: When `true`, incomplete tuples (i.e. groups with less than `size` items) are emitted as partial groups, otherwise they are discarded (default: `false`). This option can only be used with `size`.
 
 `size`
-: The number of items the grouped list(s) has to contain. When the specified size is reached, the tuple is emitted.
+: The required number of items for each group. When a group reaches the required size, it is emitted.
 
 `sort`
 : Defines the sorting criteria for the grouped items. Can be one of the following values:
 
   - `false`: No sorting is applied (default).
   - `true`: Order the grouped items by the item's natural ordering i.e. numerical for number, lexicographic for string, etc. See the [Java documentation](http://docs.oracle.com/javase/tutorial/collections/interfaces/order.html) for more information.
-  - `hash`: Order the grouped items by the hash number associated to each entry.
-  - `deep`: Similar to the previous, but the hash number is created on actual entries content e.g. when the item is a file, the hash is created on the actual file content.
-  - A custom sorting criteria used to order the tuples element holding list of values. It can be specified by using either a {ref}`Closure ` or a [Comparator](http://docs.oracle.com/javase/7/docs/api/java/util/Comparator.html) object.
+  - `'hash'`: Order the grouped items by the hash number associated to each entry.
+  - `'deep'`: Similar to the previous, but the hash number is created on actual entries content e.g. when the item is a file, the hash is created on the actual file content.
+  - A custom sorting criteria used to order the nested list elements of each tuple. It can be a {ref}`Closure ` or a [Comparator](http://docs.oracle.com/javase/7/docs/api/java/util/Comparator.html) object.
 
 (operator-ifempty)=
 
@@ -681,9 +698,7 @@ Available options:
 
 *Returns: value channel*
 
-The `ifEmpty` operator creates a channel which emits a default value, specified as the operator parameter, when the channel to which is applied is *empty* i.e. doesn't emit any value. Otherwise it will emit the same sequence of entries as the original channel.
-
-Thus, the following example prints:
+The `ifEmpty` operator emits a source channel, or a default value if the source channel is *empty* (doesn't emit any value):
 
 ```{literalinclude} snippets/ifempty-1.nf
 :language: groovy
@@ -693,8 +708,6 @@ Thus, the following example prints:
 :language: console
 ```
 
-Instead, this one prints:
-
 ```{literalinclude} snippets/ifempty-2.nf
 :language: groovy
 ```
@@ -703,9 +716,9 @@ Instead, this one prints:
 :language: console
 ```
 
-The `ifEmpty` value parameter can be defined with a {ref}`closure `. In this case the result value of the closure evaluation will be emitted when the empty condition is satisfied.
+The default value can also be a {ref}`closure `, in which case the closure is evaluated and the result is emitted when the source channel is empty.
 
-See also: {ref}`channel-empty` method.
+See also: {ref}`channel-empty` channel factory
 
 (operator-join)=
 
@@ -713,7 +726,9 @@ See also: {ref}`channel-empty` method.
 
 *Returns: queue channel*
 
-The `join` operator creates a channel that joins together the items emitted by two channels for which exists a matching key. The key is defined, by default, as the first element in each item emitted.
+The `join` operator emits the inner product of two source channels using a matching key.
+
+To be more precise, the operator transforms a sequence of tuples like *(K, V1, V2, ..)* and *(K, W1, W1, ..)* into a sequence of tuples like *(K, V1, V2, .., W1, W2, ..)*. It is equivalent to an *inner join* in SQL, or an *outer join* when `remainder` is `true`.
 
 For example:
 
@@ -725,9 +740,9 @@ For example:
 :language: console
 ```
 
-The `index` of a different matching element can be specified by using the `by` parameter.
+By default, the first element of each item is used as the key. The `by` option can be used to specify a different index, or list of indices.
 
-The `join` operator can emit all the pairs that are incomplete, i.e. the items for which a matching element is missing, by specifying the optional parameter `remainder` as shown below:
+By default, unmatched items are discarded. The `remainder` option can be used to emit them at the end:
 
 ```{literalinclude} snippets/join-with-remainder.nf
 :language: groovy
@@ -740,16 +755,18 @@ The `join` operator can emit all the pairs that are incomplete, i.e. the items f
 Available options:
 
 `by`
-: The index (zero based) of the element to be used as grouping key. A key composed by multiple elements can be defined specifying a list of indices e.g. `by: [0,2]`.
+: The zero-based index of each item to use as the matching key. Can also be a list of indices, e.g. `by: [0, 2]` (default: `[0]`).
 
 `failOnDuplicate`
-: An error is reported when the same key is found more than once.
+: When `true`, an error is reported when the operator receives multiple items from the same channel with the same key (default: `true` if {ref}`strict mode ` is enabled, `false` otherwise).
 
 `failOnMismatch`
-: An error is reported when a channel emits a value for which there isn't a corresponding element in the joining channel. This option cannot be used with `remainder`.
+: When `true`, an error is reported when the operator receives an item from one channel for which there no matching item from the other channel (default: `true` if {ref}`strict mode ` is enabled, `false` otherwise). This option cannot be used with `remainder`.
 
 `remainder`
-: When `false` incomplete tuples (i.e. with less than `size` grouped items) are discarded (default). When `true` incomplete tuples are emitted as the ending emission.
+: When `true`, unmatched items are emitted at the end, otherwise they are discarded (default: `false`). 
+
+See also: [combine](#combine), [cross](#cross)
 
 (operator-last)=
 
@@ -757,7 +774,7 @@ Available options:
 
 *Returns: value channel*
 
-The `last` operator creates a channel that only returns the last item emitted by the source channel. For example:
+The `last` operator emits the last item from a source channel:
 
 ```{literalinclude} snippets/last.nf
 :language: groovy
@@ -773,7 +790,7 @@ The `last` operator creates a channel that only returns the last item emitted by
 
 *Returns: queue channel*
 
-The `map` operator applies a function of your choosing to every item emitted by a channel, and returns the items so obtained as a new channel. The function applied is called the mapping function and is expressed with a {ref}`closure ` as shown in the example below:
+The `map` operator applies a *mapping function* to each item from a source channel:
 
 ```{literalinclude} snippets/map.nf
 :language: groovy
@@ -789,7 +806,7 @@ The `map` operator applies a function of your choosing to every item emitted by
 
 *Returns: value channel*
 
-The `max` operator waits until the source channel completes, and then emits the item that has the greatest value. For example:
+The `max` operator emits the item with the greatest value from a source channel:
 
 ```{literalinclude} snippets/max.nf
 :language: groovy
@@ -799,7 +816,9 @@ The `max` operator waits until the source channel completes, and then emits the
 :language: console
 ```
 
-An optional {ref}`closure ` parameter can be specified in order to provide a function that returns the value to be compared. The example below shows how to find the string item that has the maximum length:
+An optional {ref}`closure ` can be used to control how the items are compared. The closure can be a *mapping function*, which transforms each item before it is compared, or a *comparator function*, which defines how to compare two items more generally.
+
+The following examples show how to find the longest string in a channel:
 
 ```{literalinclude} snippets/max-with-mapper.nf
 :language: groovy
@@ -809,8 +828,6 @@ An optional {ref}`closure ` parameter can be specified in order
 :language: console
 ```
 
-Alternatively it is possible to specify a comparator function i.e. a {ref}`closure ` taking two parameters that represent two emitted items to be compared. For example:
-
 ```{literalinclude} snippets/max-with-comparator.nf
 :language: groovy
 ```
@@ -825,9 +842,7 @@ Alternatively it is possible to specify a comparator function i.e. a {ref}`closu
 
 *Returns: queue channel*
 
-The `merge` operator lets you join items emitted by two (or more) channels into a new channel.
-
-For example, the following code merges two channels together: one which emits a series of odd integers and the other which emits a series of even integers:
+The `merge` operator joins the items from two or more channels into a new channel:
 
 ```{literalinclude} snippets/merge.nf
 :language: groovy
@@ -837,7 +852,7 @@ For example, the following code merges two channels together: one which emits a
 :language: console
 ```
 
-An optional closure can be provided to customise the items emitted by the resulting merged channel. For example:
+An optional closure can be used to control how two items are merged:
 
 ```{literalinclude} snippets/merge-with-mapper.nf
 :language: groovy
@@ -859,7 +874,7 @@ You should always use a matching key (e.g. sample ID) to merge multiple channels
 
 *Returns: value channel*
 
-The `min` operator waits until the source channel completes, and then emits the item that has the lowest value. For example:
+The `min` operator emits the item with the lowest value from a source channel:
 
 ```{literalinclude} snippets/min.nf
 :language: groovy
@@ -869,7 +884,9 @@ The `min` operator waits until the source channel completes, and then emits the
 :language: console
 ```
 
-An optional {ref}`closure ` parameter can be specified in order to provide a function that returns the value to be compared. The example below shows how to find the string item that has the minimum length:
+An optional {ref}`closure ` can be used to control how the items are compared. The closure can be a *mapping function*, which transforms each item before it is compared, or a *comparator function*, which defines how to compare two items more generally.
+
+The following examples show how to find the shortest string in a channel:
 
 ```{literalinclude} snippets/min-with-mapper.nf
 :language: groovy
@@ -879,8 +896,6 @@ An optional {ref}`closure ` parameter can be specified in order
 :language: console
 ```
 
-Alternatively it is possible to specify a comparator function i.e. a {ref}`closure ` taking two parameters that represent two emitted items to be compared. For example:
-
 ```{literalinclude} snippets/min-with-comparator.nf
 :language: groovy
 ```
@@ -895,9 +910,7 @@ Alternatively it is possible to specify a comparator function i.e. a {ref}`closu
 
 *Returns: queue channel*
 
-The `mix` operator combines the items emitted by two (or more) channels into a single channel.
-
-For example:
+The `mix` operator emits the items from two or more source channels into a single output channel:
 
 ```{literalinclude} snippets/mix.nf
 :language: groovy
@@ -907,18 +920,18 @@ For example:
 :language: console
 ```
 
-:::{note}
-The items emitted by the resulting mixed channel may appear in any order, regardless of which source channel they came from. Thus, the following example could also be a possible result of the above example:
+The items in the mixed output channel may appear in any order, regardless of which source channel they came from. Thus, the previous example could also output the following:
 
-```
-'z'
+```console
+z
 1
-'a'
+a
 2
-'b'
+b
 3
 ```
-:::
+
+See also: [concat](#concat)
 
 (operator-multimap)=
 
@@ -929,9 +942,9 @@ The items emitted by the resulting mixed channel may appear in any order, regard
 
 *Returns: map of queue channels*
 
-The `multiMap` operator allows you to forward the items emitted by a source channel to two or more output channels, mapping each input value as a separate element.
+The `multiMap` operator applies a set of mapping functions to a source channel, producing a separate output channel for each mapping function.
 
-The mapping criteria is defined with a {ref}`closure ` that specifies the target channels (labelled with a unique identifier) followed by an expression that maps each item from the input channel to the target channel.
+The multi-map criteria is a {ref}`closure ` that defines, for each output channel, a label followed by a mapping expression.
 
 For example:
 
@@ -943,7 +956,7 @@ For example:
 :language: console
 ```
 
-The mapping expression can be omitted when the value to be emitted is the same as the following one. If you just need to forward the same value to multiple channels, you can use the following shorthand:
+Multiple labels can share the same mapping expression using the following shorthand:
 
 ```{literalinclude} snippets/multimap-shared.nf
 :language: groovy
@@ -953,16 +966,16 @@ The mapping expression can be omitted when the value to be emitted is the same a
 :language: console
 ```
 
-As before, this creates two channels, but now both of them receive the same source items.
+The above example creates two channels as before, but now they both receive the same items.
 
-You can use the `multiMapCriteria` method to create a multi-map criteria as a variable that can be passed as an argument to one or more `multiMap` operations, as shown below:
+You can use the `multiMapCriteria()` method to create a multi-map criteria as a variable that can be passed as an argument to any number of `multiMap` operations, as shown below:
 
 ```{literalinclude} snippets/multimap-criteria.nf
 :language: groovy
 ```
 
 :::{note}
-If you use `multiMap` to split a tuple or map into multiple channels, it is recommended that you retain a matching key (e.g. sample ID) with *each* new channel, so that you can re-combine these channels later on if needed. In general, you should not expect to be able to merge channels correctly without a matching key, due to the parallel and asynchronous nature of Nextflow pipelines.
+If you use `multiMap` to split a tuple or map into multiple channels, it is recommended that you retain a matching key (e.g. sample ID) with *each* new channel, so that you can re-combine these channels later on if needed. In general, you should not expect to be able to merge channels correctly without a matching key, due to the concurrent nature of Nextflow pipelines.
 :::
 
 (operator-randomsample)=
@@ -971,21 +984,21 @@ If you use `multiMap` to split a tuple or map into multiple channels, it is reco
 
 *Returns: queue channel*
 
-The `randomSample` operator allows you to create a channel emitting the specified number of items randomly taken from the channel to which is applied. For example:
+The `randomSample` operator emits a randomly-selected subset of items from a source channel:
 
 ```{literalinclude} snippets/random-sample.nf
 :language: groovy
 ```
 
-The above snippet will print 10 numbers in the range from 1 to 100.
+The above snippet will print 10 randomly-selected numbers between 1 and 100 (without replacement).
 
-The operator supports a second parameter that allows you to set the initial `seed` for the random number generator. By setting it, the `randomSample` operator will always return the same pseudo-random sequence. For example:
+An optional second parameter can be used to set the initial *seed* for the random number generator, which ensures that the `randomSample` operator produces the same pseudo-random sequence across runs:
 
 ```{literalinclude} snippets/random-sample-with-seed.nf
 :language: groovy
 ```
 
-The above example will print 10 random numbers in the range between 1 and 100. At each run of the script, the same sequence will be returned.
+The above example will print 10 randomly-selected numbers between 1 and 100 (without replacement). Each subsequent script execution will produce the same sequence.
 
 (operator-reduce)=
 
@@ -993,9 +1006,7 @@ The above example will print 10 random numbers in the range between 1 and 100. A
 
 *Returns: value channel*
 
-The `reduce` operator applies a function of your choosing to every item emitted by a channel. Each time this function is invoked it takes two parameters: the accumulated value and the *i-th* emitted item. The result is passed as the accumulated value to the next function call, along with the *i+1 th* item, until all the items are processed.
-
-Finally, the `reduce` operator emits the result of the last invocation of your function as the sole output.
+The `reduce` operator applies an *accumulator function* sequentially to each item in a source channel, and emits the final accumulated value. The accumulator function takes two parameters -- the accumulated value and the *i*-th emitted item -- and it should return the accumulated result, which is passed to the next invocation with the *i+1*-th item. This process is repeated for each item in the source channel.
 
 For example:
 
@@ -1007,11 +1018,7 @@ For example:
 :language: console
 ```
 
-:::{tip}
-A common use case for this operator is to use the first parameter as an accumulator and the second parameter as the `i-th` item to be processed.
-:::
-
-Optionally you can specify an initial value for the accumulator as shown below:
+By default, the first item is used as the initial accumulated value. You can optionally specify a different initial value as shown below:
 
 ```{literalinclude} snippets/reduce-with-initial-value.nf
 :language: groovy
@@ -1027,19 +1034,19 @@ Optionally you can specify an initial value for the accumulator as shown below:
 
 *Returns: nothing*
 
-The `set` operator assigns the channel to a variable whose name is specified as a closure parameter. For example:
+The `set` operator assigns a source channel to a variable, whose name is specified as a closure parameter:
 
 ```groovy
 Channel.of(10, 20, 30).set { my_channel }
 ```
 
-This is semantically equivalent to the following assignment:
+Using `set` is semantically equivalent to assigning a variable:
 
 ```groovy
 my_channel = Channel.of(10, 20, 30)
 ```
 
-However the `set` operator is more idiomatic in Nextflow scripting, since it can be used at the end of a chain of operator transformations, thus resulting in a more fluent and readable operation.
+See also: [tap](#tap)
 
 (operator-splitcsv)=
 
@@ -1047,9 +1054,9 @@ However the `set` operator is more idiomatic in Nextflow scripting, since it can
 
 *Returns: queue channel*
 
-The `splitCsv` operator allows you to parse text items emitted by a channel, that are formatted using the [CSV format](http://en.wikipedia.org/wiki/Comma-separated_values), and split them into records or group them into list of records with a specified length.
+The `splitCsv` operator parses and splits [CSV-formatted](http://en.wikipedia.org/wiki/Comma-separated_values) text from a source channel into records, or groups of records with a given size.
 
-In the simplest case just apply the `splitCsv` operator to a channel emitting a CSV formatted text files or text entries. For example:
+For example:
 
 ```{literalinclude} snippets/splitcsv.nf
 :language: groovy
@@ -1059,9 +1066,9 @@ In the simplest case just apply the `splitCsv` operator to a channel emitting a
 :language: console
 ```
 
-The above example shows hows CSV text is parsed and is split into single rows. Values can be accessed by its column index in the row object.
+The above example shows hows CSV text is parsed and split into individual rows, where each row is simply a list of columns.
 
-When the CSV begins with a header line defining the column names, you can specify the parameter `header: true` which allows you to reference each value by its name, as shown in the following example:
+When the CSV begins with a header line defining the column names, and the `header` option is `true`, each row is returned as a map instead:
 
 ```{literalinclude} snippets/splitcsv-with-header.nf
 :language: groovy
@@ -1071,7 +1078,7 @@ When the CSV begins with a header line defining the column names, you can specif
 :language: console
 ```
 
-Alternatively you can provide custom header names by specifying a the list of strings in the `header` parameter as shown below:
+The `header` option can also just be a list of columns:
 
 ```{literalinclude} snippets/splitcsv-with-columns.nf
 :language: groovy
@@ -1081,42 +1088,37 @@ Alternatively you can provide custom header names by specifying a the list of st
 :language: console
 ```
 
-:::{note}
-- By default, the `splitCsv` operator returns each row as a *list* object. Items are accessed by using the 0-based column index.
-- When the `header` is specified each row is returned as a *map* object (also known as dictionary). Items are accessed via the corresponding column name. 
-:::
-
 Available options:
 
 `by`
-: The number of rows in each `chunk`
+: When specified, group rows into *chunks* with the given size (default: none).
 
 `charset`
-: Parse the content by using the specified charset e.g. `UTF-8`
+: Parse the content with the specified charset, e.g. `UTF-8`. See the list of [standard charsets](https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/nio/charset/StandardCharsets.html) for available options.
 
 `decompress`
-: When `true` decompress the content using the GZIP format before processing it (note: files whose name ends with `.gz` extension are decompressed automatically)
+: When `true`, decompress the content using the GZIP format before processing it (default: `false`). Files with the `.gz` extension are decompressed automatically.
 
 `elem`
-: The index of the element to split when the operator is applied to a channel emitting list/tuple objects (default: first file object or first element)
+: The index of the element to split when the source items are lists or tuples (default: first file object or first element).
 
 `header`
-: When `true` the first line is used as columns names. Alternatively it can be used to provide the list of columns names.
+: When `true`, the first line is used as the columns names (default: `false`). Can also be a list of columns names.
 
 `limit`
-: Limits the number of retrieved records for each file to the specified value.
+: Limits the number of records to retrieve for each source item (default: no limit).
 
 `quote`
-: Values may be quoted by single or double quote characters.
+: The character used to quote values (default: `''` or `""`).
 
 `sep`
-: The character used to separate the values (default: `,`)
+: The character used to separate values (default: `,`)
 
 `skip`
-: Number of lines since the file beginning to ignore when parsing the CSV content.
+: Number of lines to ignore from the beginning when parsing the CSV text (default: `0`).
 
 `strip`
-: Removes leading and trailing blanks from values (default: `false`)
+: When `true`, remove leading and trailing blanks from values (default: `false`).
 
 (operator-splitfasta)=
 
@@ -1124,9 +1126,9 @@ Available options:
 
 *Returns: queue channel*
 
-The `splitFasta` operator allows you to split the entries emitted by a channel, that are formatted using the [FASTA format](http://en.wikipedia.org/wiki/FASTA_format). It returns a channel which emits text item for each sequence in the received FASTA content.
+The `splitFasta` operator splits [FASTA-formatted](http://en.wikipedia.org/wiki/FASTA_format) text from a source channel into individual sequences.
 
-The number of sequences in each text chunk produced by the `splitFasta` operator can be set by using the `by` parameter. The following example shows how to read a FASTA file and split it into chunks containing 10 sequences each:
+The `by` option can be used to group sequences into chunks of a given size. The following example shows how to read a FASTA file and split it into chunks of 10 sequences each:
 
 ```groovy
 Channel
@@ -1136,59 +1138,57 @@ Channel
 ```
 
 :::{warning}
-Chunks are stored in memory by default. When splitting large files, specify the parameter `file: true` to save the chunks into files in order to avoid an `OutOfMemoryException`. See the parameter table below for details.
+Chunks are stored in memory by default. When splitting large files, specify `file: true` to save the chunks into files in order to avoid running out of memory. See the list of options below for details.
 :::
 
-A second version of the `splitFasta` operator allows you to split a FASTA content into record objects, instead of text chunks. A record object contains a set of fields that let you access and manipulate the FASTA sequence information with ease.
-
-In order to split a FASTA content into record objects, simply use the `record` parameter specifying the map of required the fields, as shown in the example below:
+The `record` option can be used to split FASTA content into *records* instead of text chunks. Each record is a map that allows you to access the FASTA sequence data with ease. For example:
 
 ```groovy
 Channel
      .fromPath('misc/sample.fa')
-     .splitFasta( record: [id: true, seqString: true ])
+     .splitFasta( record: [id: true, seqString: true] )
      .filter { record -> record.id =~ /^ENST0.*/ }
      .view { record -> record.seqString }
 ```
 
-In this example, the file `misc/sample.fa` is split into records containing the `id` and the `seqString` fields (i.e. the sequence id and the sequence data). The following `filter` operator only keeps the sequences whose ID starts with the `ENST0` prefix, finally the sequence content is printed by using the `subscribe` operator.
+The above example loads the `misc/sample.fa` file, splits it into records containing the `id` and `seqString` fields (i.e. the sequence id and the sequence data), filters records by their ID, and finally prints the sequence string of each record.
 
 Available options:
 
 `by`
-: Defines the number of sequences in each `chunk` (default: `1`)
+: Defines the number of sequences in each chunk (default: `1`).
 
 `charset`
-: Parse the content by using the specified charset e.g. `UTF-8`.
+: Parse the content with the specified charset, e.g. `UTF-8`. See the list of [standard charsets](https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/nio/charset/StandardCharsets.html) for available options.
 
 `compress`
-: When `true` resulting file chunks are GZIP compressed. The `.gz` suffix is automatically added to chunk file names.
+: When `true`, resulting file chunks are GZIP compressed (default: `false`). The `.gz` suffix is automatically added to chunk file names.
 
 `decompress`
-: When `true`, decompress the content using the GZIP format before processing it (note: files whose name ends with `.gz` extension are decompressed automatically).
+: When `true`, decompress the content using the GZIP format before processing it (default: `false`). Files with the `.gz` extension are decompressed automatically.
 
 `elem`
-: The index of the element to split when the operator is applied to a channel emitting list/tuple objects (default: first file object or first element).
+: The index of the element to split when the source items are lists or tuples (default: first file object or first element).
 
 `file`
-: When `true` saves each split to a file. Use a string instead of `true` value to create split files with a specific name (split index number is automatically added). Finally, set this attribute to an existing directory, in order to save the split files into the specified folder.
+: When `true`, saves each split to a file. Use a string instead of `true` value to create split files with a specific name (split index number is automatically added). Finally, set this attribute to an existing directory, in order to save the split files into the specified directory.
 
 `limit`
-: Limits the number of retrieved sequences for each file to the specified value.
+: Limits the number of sequences to retrieve for each source item (default: no limit).
 
 `record`
-: Parse each entry in the FASTA file as record objects. The following fields are available:
+: Parse each entry in the FASTA file into a record. The following fields are available:
 
-  - `id`: The FASTA sequence identifier i.e. the word following the `>` symbol up to the first `blank` or `newline` character
+  - `id`: The FASTA sequence identifier, i.e. the word following the `>` symbol up to the first blank or newline character
   - `header`: The first line in a FASTA sequence without the `>` character
   - `desc`: The text in the FASTA header following the ID value
   - `text`: The complete FASTA sequence including the header
-  - `seqString`: The sequence data as a single line string i.e. containing no `newline` characters
-  - `sequence`: The sequence data as a multi-line string (always ending with a `newline` character)
-  - `width`: Define the length of a single line when the `sequence` field is used, after that the sequence data continues on a new line.
+  - `seqString`: The sequence data as a single-line string, i.e. containing no newline characters
+  - `sequence`: The sequence data as a multi-line string, i.e. always ending with a newline character
+  - `width`: Define the length of a single line when the `sequence` field is used, after which the sequence data continues on a new line.
 
 `size`
-: Defines the size in memory units of the expected chunks e.g. `1.MB`.
+: Defines the size of the expected chunks as a memory unit, e.g. `1.MB`.
 
 See also: [countFasta](#countfasta)
 
@@ -1198,9 +1198,9 @@ See also: [countFasta](#countfasta)
 
 *Returns: queue channel*
 
-The `splitFastq` operator allows you to split the entries emitted by a channel, that are formatted using the [FASTQ format](http://en.wikipedia.org/wiki/FASTQ_format). It returns a channel which emits a text chunk for each sequence in the received item.
+The `splitFasta` operator splits [FASTQ formatted](http://en.wikipedia.org/wiki/FASTQ_format) text from a source channel into individual sequences.
 
-The number of sequences in each text chunk produced by the `splitFastq` operator is defined by the parameter `by`. The following example shows you how to read a FASTQ file and split it into chunks containing 10 sequences each:
+The `by` option can be used to group sequences into chunks of a given size. The following example shows how to read a FASTQ file and split it into chunks of 10 sequences each:
 
 ```groovy
 Channel
@@ -1210,12 +1210,10 @@ Channel
 ```
 
 :::{warning}
-Chunks are stored in memory by default. When splitting large files, specify the parameter `file: true` to save the chunks into files in order to avoid an `OutOfMemoryException`. See the parameter table below for details.
+Chunks are stored in memory by default. When splitting large files, specify `file: true` to save the chunks into files in order to avoid running out of memory. See the list of options below for details.
 :::
 
-A second version of the `splitFastq` operator allows you to split a FASTQ formatted content into record objects, instead of text chunks. A record object contains a set of fields that let you access and manipulate the FASTQ sequence data with ease.
-
-In order to split FASTQ sequences into record objects simply use the `record` parameter specifying the map of the required fields, or just specify `record: true` as in the example shown below:
+The `record` option can be used to split FASTQ content into *records* instead of text chunks. Each record is a map that allows you to access the FASTQ sequence data with ease. For example:
 
 ```groovy
 Channel
@@ -1224,7 +1222,7 @@ Channel
     .view { record -> record.readHeader }
 ```
 
-Finally the `splitFastq` operator is able to split paired-end read pair FASTQ files. It must be applied to a channel which emits tuples containing at least two elements that are the files to be split. For example:
+The `pe` option can be used to split paired-end FASTQ files. The source channel must emit tuples containing the file pairs. For example:
 
 ```groovy
 Channel
@@ -1234,41 +1232,41 @@ Channel
 ```
 
 :::{note}
-The `fromFilePairs` requires the `flat: true` option in order to emit the file pairs as separate elements in the produced tuples.
+`Channel.fromFilePairs()` requires the `flat: true` option in order to emit the file pairs as separate elements in the produced tuples.
 :::
 
 :::{note}
-This operator assumes that the order of the paired-end reads correspond with each other and both files contain the same number of reads.
+This operator assumes that the order of the paired-end reads correspond with each other and that both files contain the same number of reads.
 :::
 
 Available options:
 
 `by`
-: Defines the number of *reads* in each `chunk` (default: `1`)
+: Defines the number of sequences in each chunk (default: `1`).
 
 `charset`
-: Parse the content by using the specified charset e.g. `UTF-8`
+: Parse the content with the specified charset, e.g. `UTF-8`. See the list of [standard charsets](https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/nio/charset/StandardCharsets.html) for available options.
 
 `compress`
-: When `true` resulting file chunks are GZIP compressed. The `.gz` suffix is automatically added to chunk file names.
+: When `true`, resulting file chunks are GZIP compressed (default: `false`). The `.gz` suffix is automatically added to chunk file names.
 
 `decompress`
-: When `true` decompress the content using the GZIP format before processing it (note: files whose name ends with `.gz` extension are decompressed automatically)
+: When `true`, decompress the content using the GZIP format before processing it (default: `false`). Files with the `.gz` extension are decompressed automatically.
 
 `elem`
-: The index of the element to split when the operator is applied to a channel emitting list/tuple objects (default: first file object or first element)
+: The index of the element to split when the source items are lists or tuples (default: first file object or first element).
 
 `file`
-: When `true` saves each split to a file. Use a string instead of `true` value to create split files with a specific name (split index number is automatically added). Finally, set this attribute to an existing directory, in order to save the split files into the specified folder.
+: When `true`, saves each split to a file. Use a string instead of `true` value to create split files with a specific name (split index number is automatically added). Finally, set this attribute to an existing directory, in order to save the split files into the specified directory.
 
 `limit`
-: Limits the number of retrieved *reads* for each file to the specified value.
+: Limits the number of sequences to retrieve for each source item (default: no limit).
 
 `pe`
-: When `true` splits paired-end read files, therefore items emitted by the source channel must be tuples in which at least two elements are the read-pair files to be split.
+: When `true`, splits paired-end read files. Items emitted by the source channel must be tuples with the file pairs.
 
 `record`
-: Parse each entry in the FASTQ file as record objects. The following fields are available:
+: Parse each entry in the FASTQ file into a record. The following fields are available:
 
   - `readHeader`: Sequence header (without the `@` prefix)
   - `readString`: The raw sequence data
@@ -1283,9 +1281,9 @@ See also: [countFastq](#countfastq)
 
 *Returns: queue channel*
 
-The `splitJson` operator allows you to split a JSON document from a source channel into individual records. If the document is a JSON array, each element of the array will be emitted. If the document is a JSON object, each key-value pair will be emitted as a map with the properties `key`  and `value`.
+The `splitJson` operator splits [JSON formatted](https://en.wikipedia.org/wiki/JSON) text from a source channel into individual records.
 
-An example with a JSON array:
+If the source item is a JSON array, each element of the array will be emitted:
 
 ```{literalinclude} snippets/splitjson-array.nf
 :language: groovy
@@ -1295,7 +1293,7 @@ An example with a JSON array:
 :language: console
 ```
 
-An example with a JSON object:
+If the source item is a JSON object, each key-value pair will be emitted as a map with the properties `key`  and `value`:
 
 ```{literalinclude} snippets/splitjson-object.nf
 :language: groovy
@@ -1305,7 +1303,7 @@ An example with a JSON object:
 :language: console
 ```
 
-You can optionally query a section of the JSON document to parse and split, using the `path` option:
+The `path` option can be used to query a section of the JSON document to parse and split:
 
 ```{literalinclude} snippets/splitjson-with-path.nf
 :language: groovy
@@ -1318,10 +1316,10 @@ You can optionally query a section of the JSON document to parse and split, usin
 Available options:
 
 `limit`
-: Limits the number of retrieved lines for each file to the specified value.
+: Limits the number of records to retrieve for each source item (default: no limit).
 
 `path`
-: Define the section of the JSON document that you want to extract. The expression is a set of paths separated by a dot, similar to [JSONPath](https://goessner.net/articles/JsonPath/). The empty string is the document root (default). An integer in brackets is the 0-based index in a JSON array. A string preceded by a dot `.` is the key in a JSON object.
+: Defines a query for a section of each source item to parse and split. The expression should be a path similar to [JSONPath](https://goessner.net/articles/JsonPath/). The empty string is the document root (default). An integer in brackets is a zero-based index in a JSON array. A string preceded by a dot `.` is a key in a JSON object.
 
 See also: [countJson](#countjson)
 
@@ -1331,9 +1329,7 @@ See also: [countJson](#countjson)
 
 *Returns: queue channel*
 
-The `splitText` operator allows you to split multi-line strings or text file items, emitted by a source channel into chunks containing `n` lines, which will be emitted by the resulting channel.
-
-For example:
+The `splitText` operator splits multi-line text content from a source channel into chunks of *N* lines:
 
 ```groovy
 Channel
@@ -1342,9 +1338,9 @@ Channel
     .view()
 ```
 
-It splits the content of the files with suffix `.txt`, and prints it line by line.
+The above example loads a collection of text files, splits the content of each file into individual lines, and prints each line.
 
-By default the `splitText` operator splits each item into chunks of one line. You can define the number of lines in each chunk by using the parameter `by`, as shown in the following example:
+The `by` option can be used to emit chunks of *N* lines:
 
 ```groovy
 Channel
@@ -1356,7 +1352,7 @@ Channel
     }
 ```
 
-An optional {ref}`closure ` can be specified in order to transform the text chunks produced by the operator. The following example shows how to split text files into chunks of 10 lines and transform them to capital letters:
+An optional {ref}`closure ` can be used to transform each text chunk produced by the operator. The following example shows how to split text files into chunks of 10 lines and transform them to uppercase letters:
 
 ```groovy
 Channel
@@ -1375,25 +1371,25 @@ Available options:
 : Defines the number of lines in each `chunk` (default: `1`).
 
 `charset`
-: Parse the content by using the specified charset e.g. `UTF-8`.
+: Parse the content with the specified charset, e.g. `UTF-8`. See the list of [standard charsets](https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/nio/charset/StandardCharsets.html) for available options.
 
 `compress`
-: When `true` resulting file chunks are GZIP compressed. The `.gz` suffix is automatically added to chunk file names.
+: When `true`, resulting file chunks are GZIP compressed (default: `false`). The `.gz` suffix is automatically added to chunk file names.
 
 `decompress`
-: When `true`, decompress the content using the GZIP format before processing it (note: files whose name ends with `.gz` extension are decompressed automatically).
+: When `true`, decompresses the content using the GZIP format before processing it (default: `false`). Files with the `.gz` extension are decompressed automatically.
 
 `elem`
-: The index of the element to split when the operator is applied to a channel emitting list/tuple objects (default: first file object or first element).
+: The index of the element to split when the source items are lists or tuples (default: first file object or first element).
 
 `file`
-: When `true` saves each split to a file. Use a string instead of `true` value to create split files with a specific name (split index number is automatically added). Finally, set this attribute to an existing directory, in order to save the split files into the specified folder.
+: When `true`, saves each split to a file. Use a string instead of `true` value to create split files with a specific name (split index number is automatically added). Finally, set this attribute to an existing directory, in order to save the split files into the specified directory.
 
 `keepHeader`
-: Parses the first line as header and prepends it to each emitted chunk.
+: Parses the first line as header and prepends it to each emitted chunk (default: `false`).
 
 `limit`
-: Limits the number of retrieved lines for each file to the specified value.
+: Limits the number of lines to retrieve for each source item (default: no limit).
 
 See also: [countLines](#countlines)
 
@@ -1403,9 +1399,7 @@ See also: [countLines](#countlines)
 
 *Returns: nothing*
 
-The `subscribe` operator allows you to execute a user defined function each time a new value is emitted by the source channel.
-
-The emitted value is passed implicitly to the specified function. For example:
+The `subscribe` operator invokes a custom function for each item from a source channel:
 
 ```{literalinclude} snippets/subscribe.nf
 :language: groovy
@@ -1415,11 +1409,7 @@ The emitted value is passed implicitly to the specified function. For example:
 :language: console
 ```
 
-:::{note}
-In Groovy, the language on which Nextflow is based, the user defined function is called a **closure**. Read the {ref}`script-closure` section to learn more about closures.
-:::
-
-If needed the closure parameter can be defined explicitly, using a name other than `it` and, optionally, specifying the expected value type, as shown in the following example:
+The closure parameter can be defined explicitly if needed, using a name other than `it` and, optionally, the expected type:
 
 ```{literalinclude} snippets/subscribe-with-param.nf
 :language: groovy
@@ -1429,16 +1419,7 @@ If needed the closure parameter can be defined explicitly, using a name other th
 :language: console
 ```
 
-```
-```
-
-The `subscribe` operator may accept one or more of the following event handlers:
-
-- `onNext`: function that is invoked whenever the channel emits a value. Equivalent to using the `subscribe` with a plain closure as described in the examples above.
-- `onComplete`: function that is invoked after the last value is emitted by the channel.
-- `onError`: function that it is invoked when an exception is raised while handling the `onNext` event. It will not make further calls to `onNext` or `onComplete`. The `onError` method takes as its parameter the `Throwable` that caused the error.
-
-For example:
+The `subscribe` operator supports multiple types of event handlers:
 
 ```{literalinclude} snippets/subscribe-with-on-complete.nf
 :language: groovy
@@ -1448,13 +1429,24 @@ For example:
 :language: console
 ```
 
+Available options:
+
+`onNext`
+: Closure that is invoked when an item is emitted. Equivalent to providing a closure as the first argument.
+
+`onComplete`
+: Closure that is invoked after the last item is emitted by the channel.
+
+`onError`
+: Closure that is invoked when an exception is raised while handling the `onNext` event. It will not make further calls to `onNext` or `onComplete`. The `onError` method takes as its parameter the `Throwable` that caused the error.
+
 (operator-sum)=
 
 ## sum
 
 *Returns: value channel*
 
-The `sum` operator creates a channel that emits the sum of all the items emitted by the channel itself. For example:
+The `sum` operator emits the sum of all items in a source channel:
 
 ```{literalinclude} snippets/sum.nf
 :language: groovy
@@ -1464,7 +1456,7 @@ The `sum` operator creates a channel that emits the sum of all the items emitted
 :language: console
 ```
 
-An optional {ref}`closure ` parameter can be specified in order to provide a function that, given an item, returns the value to be summed. For example:
+An optional {ref}`closure ` can be used to transform each item before it is added to the sum:
 
 ```{literalinclude} snippets/sum-with-mapper.nf
 :language: groovy
@@ -1478,7 +1470,7 @@ An optional {ref}`closure ` parameter can be specified in order
 
 *Returns: queue channel*
 
-The `take` operator allows you to filter only the first `n` items emitted by a channel. For example:
+The `take` operator takes the first *N* items from a source channel:
 
 ```{literalinclude} snippets/take.nf
 :language: groovy
@@ -1492,15 +1484,13 @@ The `take` operator allows you to filter only the first `n` items emitted by a c
 Specifying a size of `-1` causes the operator to take all values.
 :::
 
-See also [until](#until).
+See also: [until](#until)
 
 ## tap
 
 *Returns: queue channel*
 
-The `tap` operator is like the [set](#set) operator in that it assigns a source channel to a new target channel.
-but it also emits the source channel for downstream use. This operator is a useful way to extract intermediate
-output channels from a chain of operators. For example:
+The `tap` operator assigns a source channel to a variable, and emits the source channel. It is a useful way to extract intermediate output channels from a chain of operators. For example:
 
 ```{literalinclude} snippets/tap.nf
 :language: groovy
@@ -1510,11 +1500,13 @@ output channels from a chain of operators. For example:
 :language: console
 ```
 
+See also: [set](#set)
+
 ## toInteger
 
 *Returns: queue channel*
 
-The `toInteger` operator allows you to convert the string values emitted by a channel to `Integer` values. For example:
+The `toInteger` operator converts string values from a source channel to integer values:
 
 ```{literalinclude} snippets/tointeger.nf
 :language: groovy
@@ -1524,7 +1516,15 @@ The `toInteger` operator allows you to convert the string values emitted by a ch
 :language: console
 ```
 
-:::{tip}
+:::{note}
+`toInteger` is equivalent to:
+
+```groovy
+map { it -> it as Integer }
+```
+:::
+
+:::{note}
 You can also use `toLong`, `toFloat`, and `toDouble` to convert to other numerical types.
 :::
 
@@ -1532,7 +1532,7 @@ You can also use `toLong`, `toFloat`, and `toDouble` to convert to other numeric
 
 *Returns: value channel*
 
-The `toList` operator collects all the items emitted by a channel to a `List` object and emits the resulting collection as a single item. For example:
+The `toList` operator collects all the items from a source channel into a list and emits the list as a single item:
 
 ```{literalinclude} snippets/tolist.nf
 :language: groovy
@@ -1543,7 +1543,7 @@ The `toList` operator collects all the items emitted by a channel to a `List` ob
 ```
 
 :::{note}
-There are two differences between `toList` and `collect`:
+There are two main differences between `toList` and `collect`:
 
 - When there is no input, `toList` emits an empty list whereas `collect` emits nothing.
 - By default, `collect` flattens list items by one level.
@@ -1555,13 +1555,13 @@ collect(flat: false).ifEmpty([])
 ```
 :::
 
-See also: [collect](#collect) operator.
+See also: [collect](#collect)
 
 ## toSortedList
 
 *Returns: value channel*
 
-The `toSortedList` operator collects all the items emitted by a channel to a `List` object where they are sorted and emits the resulting collection as a single item. For example:
+The `toSortedList` operator collects all the items from a source channel into a sorted list and emits the list as a single item:
 
 ```{literalinclude} snippets/tosortedlist.nf
 :language: groovy
@@ -1571,8 +1571,7 @@ The `toSortedList` operator collects all the items emitted by a channel to a `Li
 :language: console
 ```
 
-You may also pass a comparator closure as an argument to the `toSortedList` operator to customize the sorting criteria. For example, to sort by the second element of a tuple in descending order:
-
+An optional closure can be used to control how items are compared when sorting. For example, to sort tuples by their second element in descending order:
 
 ```{literalinclude} snippets/tosortedlist-with-comparator.nf
 :language: groovy
@@ -1582,78 +1581,69 @@ You may also pass a comparator closure as an argument to the `toSortedList` oper
 :language: console
 ```
 
-See also: [collect](#collect) operator.
+:::{note}
+`toSortedList` is equivalent to:
+
+```groovy
+collect(flat: false, sort: true).ifEmpty([])
+```
+:::
+
+See also: [collect](#collect)
 
 ## transpose
 
 *Returns: queue channel*
 
-The `transpose` operator transforms a channel in such a way that the emitted items are the result of a transposition of all tuple elements in each item. For example:
+The `transpose` operator "transposes" each tuple from a source channel by flattening any nested list in each tuple, emitting each nested item separately.
+
+To be more precise, the operator transforms a sequence of tuples like *(K, list(V), list(W), ..)* into a sequence of tuples like *(K, V, W, ..)*.
 
-```{literalinclude} snippets/transpose.nf
+For example:
+
+```{literalinclude} snippets/transpose-1.nf
 :language: groovy
 ```
 
-```{literalinclude} snippets/transpose.out
+```{literalinclude} snippets/transpose-1.out
 :language: console
 ```
 
-If each element of the channel has more than 2 items, these will be flattened by the first item in the element and only emit an element when the element is complete:
+If each source item has more than two elements, these will be flattened by the first element in the item, and a new item will be emitted only when it is complete:
 
-```groovy
-Channel.of(
-        [1, [1], ['A']],
-        [2, [1, 2], ['B', 'C']],
-        [3, [1, 2, 3], ['D', 'E']]
-    )
-    .transpose()
-    .view()
+```{literalinclude} snippets/transpose-2.nf
+:language: groovy
 ```
 
-```
-[1, 1, A]
-[2, 1, B]
-[2, 2, C]
-[3, 1, D]
-[3, 2, E]
+```{literalinclude} snippets/transpose-2.out
+:language: console
 ```
 
-To emit all elements, use `remainder: true`:
+The `remainder` option can be used to emit any incomplete items:
 
-```groovy
-Channel.of(
-        [1, [1], ['A']],
-        [2, [1, 2], ['B', 'C']],
-        [3, [1, 2, 3], ['D', 'E']]
-    )
-    .transpose(remainder: true)
-    .view()
+```{literalinclude} snippets/transpose-2-with-remainder.nf
+:language: groovy
 ```
 
-```
-[1, 1, A]
-[2, 1, B]
-[2, 2, C]
-[3, 1, D]
-[3, 2, E]
-[3, 3, null]
+```{literalinclude} snippets/transpose-2-with-remainder.out
+:language: console
 ```
 
 Available options:
 
-` by`
-: The index (zero based) of the element to be transposed. Multiple elements can be defined specifying as list of indices e.g. `by: [0,2]`
+`by`
+: The zero-based index of the element to be transposed. Can also be a list of indices, e.g. `by: [0,2]`. By default, every list element is transposed.
+
+`remainder`
+: When `true`, incomplete tuples are emitted with `null` values for missing elements, otherwise they are discarded (default: `false`). 
 
-` remainder`
-: When `false` incomplete tuples are discarded (default). When `true` incomplete tuples are emitted containing a `null` in place of a missing element.
+See also: [groupTuple](#grouptuple)
 
 ## unique
 
 *Returns: queue channel*
 
-The `unique` operator allows you to remove duplicate items from a channel and only emit single items with no repetition.
-
-For example:
+The `unique` operator emits the unique items from a source channel:
 
 ```{literalinclude} snippets/unique.nf
 :language: groovy
@@ -1663,7 +1653,7 @@ For example:
 :language: console
 ```
 
-You can also specify an optional {ref}`closure ` that customizes the way it distinguishes between unique items. For example:
+An optional {ref}`closure ` can be used to transform each item before it is evaluated for uniqueness:
 
 ```{literalinclude} snippets/unique-with-mapper.nf
 :language: groovy
@@ -1673,11 +1663,17 @@ You can also specify an optional {ref}`closure ` that customizes
 :language: console
 ```
 
+:::{note}
+The difference between `unique` and `distinct` is that `unique` removes *all* duplicate values, whereas `distinct` removes only *consecutive* duplicate values. As a result, `unique` must process the entire source channel before it can emit anything, whereas `distinct` can emit each value immediately.
+:::
+
+See also: [distinct](#distinct)
+
 ## until
 
 *Returns: queue channel*
 
-The `until` operator creates a channel that returns the items emitted by the source channel and stop when the condition specified is verified. For example:
+The `until` operator emits each item from a source channel until a stopping condition is satisfied:
 
 ```{literalinclude} snippets/until.nf
 :language: groovy
@@ -1687,7 +1683,7 @@ The `until` operator creates a channel that returns the items emitted by the sou
 :language: console
 ```
 
-See also [take](#take).
+See also: [take](#take)
 
 (operator-view)=
 
@@ -1695,7 +1691,7 @@ See also [take](#take).
 
 *Returns: queue channel*
 
-The `view` operator prints the items emitted by a channel to the console standard output. For example:
+The `view` operator prints each item from a source channel to standard output:
 
 ```{literalinclude} snippets/view.nf
 :language: groovy
@@ -1705,9 +1701,7 @@ The `view` operator prints the items emitted by a channel to the console standar
 :language: console
 ```
 
-Each item is printed on a separate line unless otherwise specified by using the `newLine: false` optional parameter.
-
-How the channel items are printed can be controlled by using an optional closure parameter. The closure must return the actual value of the item to be printed:
+An optional closure can be used to transform each item before it is printed:
 
 ```{literalinclude} snippets/view-with-mapper.nf
 :language: groovy
@@ -1718,3 +1712,8 @@ How the channel items are printed can be controlled by using an optional closure
 ```
 
 The `view` operator also emits every item that it receives, allowing it to be chained with other operators.
+
+Available options:
+
+`newLine`
+: Print each item to a separate line (default: `true`).
diff --git a/docs/process.md b/docs/process.md
index 73d6dec191..57062d19f1 100644
--- a/docs/process.md
+++ b/docs/process.md
@@ -1166,62 +1166,59 @@ process foo {
 ```
 :::
 
-### Optional outputs
+(process-additional-options)=
 
-In most cases, a process is expected to produce an output for each output definition. However, there are situations where it is valid for a process to not generate output. In these cases, `optional: true` may be added to the output definition, which tells Nextflow not to fail the process if the declared output is not produced:
+### Additional options
 
-```groovy
-output:
-    path("output.txt"), optional: true
-```
+The following options are available for all process outputs:
 
-In this example, the process is normally expected to produce an `output.txt` file, but in the cases where the file is legitimately missing, the process does not fail. The output channel will only contain values for those processes that produce `output.txt`.
+`emit: `
 
-(process-multiple-outputs)=
+: Defines the name of the output channel, which can be used to access the channel by name from the process output:
 
-### Multiple outputs
+  ```groovy
+  process FOO {
+      output:
+      path 'hello.txt', emit: hello
+      path 'bye.txt', emit: bye
+
+      """
+      echo "hello" > hello.txt
+      echo "bye" > bye.txt
+      """
+  }
 
-When a process declares multiple outputs, each output can be accessed by index. The following example prints the second process output (indexes start at zero):
+  workflow {
+      FOO()
+      FOO.out.hello.view()
+  }
+  ```
 
-```groovy
-process FOO {
-    output:
-    path 'bye_file.txt'
-    path 'hi_file.txt'
+  See {ref}`workflow-process-invocation` for more details.
 
-    """
-    echo "bye" > bye_file.txt
-    echo "hi" > hi_file.txt
-    """
-}
+`optional: true | false`
 
-workflow {
-    FOO()
-    FOO.out[1].view()
-}
-```
+: Normally, if a specified output is not produced by the task, the task will fail. Setting `optional: true` will cause the task to not fail, and instead emit nothing to the given output channel.
 
-You can also use the `emit` option to assign a name to each output and access them by name:
+  ```groovy
+  output:
+  path("output.txt"), optional: true
+  ```
 
-```groovy
-process FOO {
-    output:
-    path 'bye_file.txt', emit: bye_file
-    path 'hi_file.txt',  emit: hi_file
+  In this example, the process is normally expected to produce an `output.txt` file, but in the cases where the file is missing, the task will not fail. The output channel will only contain values for those tasks that produced `output.txt`.
 
-    """
-    echo "bye" > bye_file.txt
-    echo "hi" > hi_file.txt
-    """
-}
+: :::{note}
+  While this option can be used with any process output, it cannot be applied to individual elements of a [tuple](#output-type-tuple) output. The entire tuple must be optional or not optional.
+  :::
 
-workflow {
-    FOO()
-    FOO.out.hi_file.view()
-}
-```
+`topic: `
+
+: :::{versionadded} 23.11.0-edge
+  :::
+
+: *Experimental: may change in a future release.*
 
-See {ref}`workflow-process-invocation` for more details.
+: Defines the {ref}`channel topic ` to which the output will be sent.
 
 ## When
 
@@ -2005,7 +2002,7 @@ process your_task {
 
 The above snippet defines an environment variable named `FOO` whose value is `bar`.
 
-When defined in the Nextflow configuration file, pod settings should be defined as maps. For example:
+When defined in the Nextflow configuration file, a pod setting can be defined as a map:
 
 ```groovy
 process {
@@ -2013,13 +2010,13 @@ process {
 }
 ```
 
-Multiple pod settings can be provided as a list of maps:
+Or as a list of maps:
 
 ```groovy
 process {
   pod = [
-      [env: 'FOO', value: 'bar'],
-      [secret: 'my-secret/key1', mountPath: '/etc/file.txt']
+    [env: 'FOO', value: 'bar'],
+    [secret: 'my-secret/key1', mountPath: '/etc/file.txt']
   ]
 }
 ```
diff --git a/docs/script.md b/docs/script.md
index 071675e103..f8471f76d4 100644
--- a/docs/script.md
+++ b/docs/script.md
@@ -86,7 +86,7 @@ Learn more about lists:
 Maps are used to store *associative arrays* (also known as *dictionaries*). They are unordered collections of heterogeneous, named data:
 
 ```groovy
-scores = [ "Brett":100, "Pete":"Did not finish", "Andrew":86.87934 ]
+scores = ["Brett": 100, "Pete": "Did not finish", "Andrew": 86.87934]
 ```
 
 Note that each of the values stored in the map can be of a different type. `Brett` is an integer, `Pete` is a string, and `Andrew` is a floating-point number.
@@ -454,6 +454,50 @@ Local variables should be declared using a qualifier such as `def` or a type nam
 
 Learn more about closures in the [Groovy documentation](http://groovy-lang.org/closures.html)
 
+### Syntax sugar
+
+Groovy provides several forms of "syntax sugar", or shorthands that can make your code easier to read.
+
+Some programming languages require every statement to be terminated by a semi-colon. In Groovy, semi-colons are optional, but they can still be used to write multiple statements on the same line:
+
+```groovy
+println 'Hello!' ; println 'Hello again!'
+```
+
+When calling a function, the parentheses around the function arguments are optional:
+
+```groovy
+// full syntax
+printf('Hello %s!\n', 'World')
+
+// shorthand
+printf 'Hello %s!\n', 'World'
+```
+
+It is especially useful when calling a function with a closure parameter:
+
+```groovy
+// full syntax
+[1, 2, 3].each({ println it })
+
+// shorthand
+[1, 2, 3].each { println it }
+```
+
+If the last argument is a closure, the closure can be written outside of the parentheses:
+
+```groovy
+// full syntax
+[1, 2, 3].inject('result:', { accum, v -> accum + ' ' + v })
+
+// shorthand
+[1, 2, 3].inject('result:') { accum, v -> accum + ' ' + v }
+```
+
+:::{note}
+In some cases, you might not be able to omit the parentheses because it would be syntactically ambiguous. You can use the `groovysh` REPL console to play around with Groovy and figure out what works.
+:::
+
 (implicit-variables)=
 
 ## Implicit variables
diff --git a/docs/snippets/transpose.nf b/docs/snippets/transpose-1.nf
similarity index 100%
rename from docs/snippets/transpose.nf
rename to docs/snippets/transpose-1.nf
diff --git a/docs/snippets/transpose.out b/docs/snippets/transpose-1.out
similarity index 100%
rename from docs/snippets/transpose.out
rename to docs/snippets/transpose-1.out
diff --git a/docs/snippets/transpose-2-with-remainder.nf b/docs/snippets/transpose-2-with-remainder.nf
new file mode 100644
index 0000000000..2b90726abe
--- /dev/null
+++ b/docs/snippets/transpose-2-with-remainder.nf
@@ -0,0 +1,7 @@
+Channel.of(
+        [1, [1], ['A']],
+        [2, [1, 2], ['B', 'C']],
+        [3, [1, 2, 3], ['D', 'E']]
+    )
+    .transpose(remainder: true)
+    .view()
\ No newline at end of file
diff --git a/docs/snippets/transpose-2-with-remainder.out b/docs/snippets/transpose-2-with-remainder.out
new file mode 100644
index 0000000000..c156136ee6
--- /dev/null
+++ b/docs/snippets/transpose-2-with-remainder.out
@@ -0,0 +1,6 @@
+[1, 1, A]
+[2, 1, B]
+[2, 2, C]
+[3, 1, D]
+[3, 2, E]
+[3, 3, null]
\ No newline at end of file
diff --git a/docs/snippets/transpose-2.nf b/docs/snippets/transpose-2.nf
new file mode 100644
index 0000000000..822c7bf45e
--- /dev/null
+++ b/docs/snippets/transpose-2.nf
@@ -0,0 +1,7 @@
+Channel.of(
+        [1, [1], ['A']],
+        [2, [1, 2], ['B', 'C']],
+        [3, [1, 2, 3], ['D', 'E']]
+    )
+    .transpose()
+    .view()
\ No newline at end of file
diff --git a/docs/snippets/transpose-2.out b/docs/snippets/transpose-2.out
new file mode 100644
index 0000000000..6c9ede99ac
--- /dev/null
+++ b/docs/snippets/transpose-2.out
@@ -0,0 +1,5 @@
+[1, 1, A]
+[2, 1, B]
+[2, 2, C]
+[3, 1, D]
+[3, 2, E]
\ No newline at end of file
diff --git a/docs/workflow.md b/docs/workflow.md
index 260ebeb37e..6bc45f1015 100644
--- a/docs/workflow.md
+++ b/docs/workflow.md
@@ -104,7 +104,7 @@ workflow {
 }
 ```
 
-When a process defines multiple output channels, each output can be accessed using the array element operator (`out[0]`, `out[1]`, etc.) or using *named outputs* (see below).
+When a process defines multiple output channels, each output can be accessed by index (`out[0]`, `out[1]`, etc.) or by name (see below).
 
 The process output(s) can also be accessed like the return value of a function:
 
@@ -144,7 +144,7 @@ workflow {
 }
 ```
 
-See {ref}`process-multiple-outputs` for more details.
+See {ref}`process outputs ` for more details.
 
 ### Process named stdout
 
diff --git a/modules/nextflow/build.gradle b/modules/nextflow/build.gradle
index 4b468ecc14..c24f235cdc 100644
--- a/modules/nextflow/build.gradle
+++ b/modules/nextflow/build.gradle
@@ -36,13 +36,13 @@ dependencies {
     api "com.beust:jcommander:1.35"
     api("com.esotericsoftware.kryo:kryo:2.24.0") { exclude group: 'com.esotericsoftware.minlog', module: 'minlog' }
     api('org.iq80.leveldb:leveldb:0.12')
-    api('org.eclipse.jgit:org.eclipse.jgit:6.5.0.202303070854-r')
+    api('org.eclipse.jgit:org.eclipse.jgit:6.6.1.202309021850-r')
     api ('javax.activation:activation:1.1.1')
     api ('javax.mail:mail:1.4.7')
     api ('org.yaml:snakeyaml:2.0')
     api ('org.jsoup:jsoup:1.15.4')
     api 'jline:jline:2.9'
-    api 'org.pf4j:pf4j:3.4.1'
+    api 'org.pf4j:pf4j:3.10.0'
     api 'dev.failsafe:failsafe:3.1.0'
 
     testImplementation 'org.subethamail:subethasmtp:3.1.7'
diff --git a/modules/nextflow/src/main/groovy/nextflow/Channel.groovy b/modules/nextflow/src/main/groovy/nextflow/Channel.groovy
index 822ce03537..84c0d2bf0e 100644
--- a/modules/nextflow/src/main/groovy/nextflow/Channel.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/Channel.groovy
@@ -121,6 +121,11 @@ class Channel  {
         return CH.queue()
     }
 
+    static DataflowWriteChannel topic(String name) {
+        if( !NF.topicChannelEnabled ) throw new MissingMethodException('topic', Channel.class, InvokerHelper.EMPTY_ARGS)
+        return CH.topic(name)
+    }
+
     /**
      * Create a empty channel i.e. only emits a STOP signal
      *
diff --git a/modules/nextflow/src/main/groovy/nextflow/NF.groovy b/modules/nextflow/src/main/groovy/nextflow/NF.groovy
index df8b592b31..3f13630088 100644
--- a/modules/nextflow/src/main/groovy/nextflow/NF.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/NF.groovy
@@ -67,4 +67,8 @@ class NF {
     static boolean isRecurseEnabled() {
         NextflowMeta.instance.preview.recursion
     }
+
+    static boolean isTopicChannelEnabled() {
+        NextflowMeta.instance.preview.topic
+    }
 }
diff --git a/modules/nextflow/src/main/groovy/nextflow/NextflowMeta.groovy b/modules/nextflow/src/main/groovy/nextflow/NextflowMeta.groovy
index b9b279fb87..bb836ab61a 100644
--- a/modules/nextflow/src/main/groovy/nextflow/NextflowMeta.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/NextflowMeta.groovy
@@ -1,5 +1,7 @@
 package nextflow
 
+import static nextflow.extension.Bolts.*
+
 import java.text.SimpleDateFormat
 import java.util.regex.Pattern
 
@@ -9,8 +11,6 @@ import groovy.transform.ToString
 import groovy.util.logging.Slf4j
 import nextflow.exception.AbortOperationException
 import nextflow.util.VersionNumber
-import static nextflow.extension.Bolts.DATETIME_FORMAT
-
 /**
  * Models nextflow script properties and metadata
  * 
@@ -43,6 +43,7 @@ class NextflowMeta {
         volatile float dsl
         boolean strict
         boolean recursion
+        boolean topic
 
         void setDsl( float num ) {
             if( num == 1 )
@@ -59,6 +60,12 @@ class NextflowMeta {
                 log.warn "NEXTFLOW RECURSION IS A PREVIEW FEATURE - SYNTAX AND FUNCTIONALITY CAN CHANGE IN FUTURE RELEASES"
             this.recursion = recurse
         }
+
+        void setTopic(Boolean value) {
+            if( topic )
+                log.warn "CHANNEL TOPICS ARE A PREVIEW FEATURE - SYNTAX AND FUNCTIONALITY CAN CHANGE IN FUTURE RELEASES"
+            this.topic = value
+        }
     }
 
     static class Features implements Flags {
@@ -80,9 +87,9 @@ class NextflowMeta {
     final Features enable = new Features()
 
     private NextflowMeta() {
-        version = new VersionNumber(Const.APP_VER)
-        build = Const.APP_BUILDNUM
-        timestamp = Const.APP_TIMESTAMP_UTC
+        version = new VersionNumber(BuildInfo.version)
+        build = BuildInfo.buildNum as int
+        timestamp = BuildInfo.timestampUTC
     }
 
     protected NextflowMeta(String ver, int build, String timestamp ) {
diff --git a/modules/nextflow/src/main/groovy/nextflow/Session.groovy b/modules/nextflow/src/main/groovy/nextflow/Session.groovy
index 2c426d672a..38945bdb3d 100644
--- a/modules/nextflow/src/main/groovy/nextflow/Session.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/Session.groovy
@@ -490,10 +490,6 @@ class Session implements ISession {
         checkConfig()
         notifyFlowBegin()
 
-        if( !NextflowMeta.instance.isDsl2() ) {
-            return
-        }
-
         // bridge any dataflow queue into a broadcast channel
         CH.broadcast()
 
@@ -1082,11 +1078,15 @@ class Session implements ISession {
     }
 
     void notifyFlowBegin() {
-        observers.each { trace -> trace.onFlowBegin() }
+        for( TraceObserver trace : observers ) {
+            trace.onFlowBegin()
+        }
     }
 
     void notifyFlowCreate() {
-        observers.each { trace -> trace.onFlowCreate(this) }
+        for( TraceObserver trace : observers ) {
+            trace.onFlowCreate(this)
+        }
     }
 
     void notifyFilePublish(Path destination, Path source=null) {
diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSLImpl.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSLImpl.groovy
index ef1b36a573..72f4a356a8 100644
--- a/modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSLImpl.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSLImpl.groovy
@@ -915,15 +915,16 @@ class NextflowDSLImpl implements ASTTransformation {
 
         /**
          * Transform a map entry `emit: something` into `emit: 'something'
+         * and `topic: something` into `topic: 'something'
          * (ie. as a constant) in a map expression passed as argument to
          * a method call. This allow the syntax
          *
          *   output:
-         *   path 'foo', emit: bar
+         *   path 'foo', emit: bar, topic: baz
          *
          * @param call
          */
-        protected void fixOutEmitOption(MethodCallExpression call) {
+        protected void fixOutEmitAndTopicOptions(MethodCallExpression call) {
             List args = isTupleX(call.arguments)?.expressions
             if( !args ) return
             if( args.size()<2 && (args.size()!=1 || call.methodAsString!='_out_stdout')) return
@@ -936,6 +937,9 @@ class NextflowDSLImpl implements ASTTransformation {
                 if( key?.text == 'emit' && val ) {
                     map.mapEntryExpressions[i] = new MapEntryExpression(key, constX(val.text))
                 }
+                else if( key?.text == 'topic' && val ) {
+                    map.mapEntryExpressions[i] = new MapEntryExpression(key, constX(val.text))
+                }
             }
         }
 
@@ -955,7 +959,7 @@ class NextflowDSLImpl implements ASTTransformation {
                 // prefix the method name with the string '_out_'
                 methodCall.setMethod( new ConstantExpression('_out_' + methodName) )
                 fixMethodCall(methodCall)
-                fixOutEmitOption(methodCall)
+                fixOutEmitAndTopicOptions(methodCall)
             }
 
             else if( methodName in ['into','mode'] ) {
diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdInfo.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/CmdInfo.groovy
index f2a174b914..e679d1fcd5 100644
--- a/modules/nextflow/src/main/groovy/nextflow/cli/CmdInfo.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/cli/CmdInfo.groovy
@@ -25,7 +25,7 @@ import com.sun.management.OperatingSystemMXBean
 import groovy.json.JsonOutput
 import groovy.transform.CompileStatic
 import groovy.util.logging.Slf4j
-import nextflow.Const
+import nextflow.BuildInfo
 import nextflow.exception.AbortOperationException
 import nextflow.plugin.DefaultPlugins
 import nextflow.plugin.Plugins
@@ -208,8 +208,8 @@ class CmdInfo extends CmdBase {
 
         def props = System.getProperties()
         def result = new StringBuilder()
-        result << BLANK << "Version: ${Const.APP_VER} build ${Const.APP_BUILDNUM}" << NEWLINE
-        result << BLANK << "Created: ${Const.APP_TIMESTAMP_UTC} ${Const.deltaLocal()}" << NEWLINE
+        result << BLANK << "Version: ${BuildInfo.version} build ${BuildInfo.buildNum}" << NEWLINE
+        result << BLANK << "Created: ${BuildInfo.timestampUTC} ${BuildInfo.timestampDelta}" << NEWLINE
         result << BLANK << "System: ${props['os.name']} ${props['os.version']}" << NEWLINE
         result << BLANK << "Runtime: Groovy ${GroovySystem.getVersion()} on ${System.getProperty('java.vm.name')} ${props['java.runtime.version']}" << NEWLINE
         result << BLANK << "Encoding: ${System.getProperty('file.encoding')} (${System.getProperty('sun.jnu.encoding')})" << NEWLINE
diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy
index 7fa7d65298..8f3edf8b06 100644
--- a/modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy
@@ -412,7 +412,7 @@ class CmdRun extends CmdBase implements HubOptions {
         // -- script can still override the DSL version
         final scriptDsl = NextflowMeta.checkDslMode(scriptText)
         if( scriptDsl ) {
-            log.debug("Applied DSL=$scriptDsl from script declararion")
+            log.debug("Applied DSL=$scriptDsl from script declaration")
             return scriptDsl
         }
         else if( dsl ) {
@@ -677,7 +677,7 @@ class CmdRun extends CmdBase implements HubOptions {
             result.putAll(json)
         }
         catch (NoSuchFileException | FileNotFoundException e) {
-            throw new AbortOperationException("Specified params file does not exists: ${file.toUriString()}")
+            throw new AbortOperationException("Specified params file does not exist: ${file.toUriString()}")
         }
         catch( Exception e ) {
             throw new AbortOperationException("Cannot parse params file: ${file.toUriString()} - Cause: ${e.message}", e)
@@ -691,7 +691,7 @@ class CmdRun extends CmdBase implements HubOptions {
             result.putAll(yaml)
         }
         catch (NoSuchFileException | FileNotFoundException e) {
-            throw new AbortOperationException("Specified params file does not exists: ${file.toUriString()}")
+            throw new AbortOperationException("Specified params file does not exist: ${file.toUriString()}")
         }
         catch( Exception e ) {
             throw new AbortOperationException("Cannot parse params file: ${file.toUriString()}", e)
diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/Launcher.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/Launcher.groovy
index 8ed41196c1..f99702b90a 100644
--- a/modules/nextflow/src/main/groovy/nextflow/cli/Launcher.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/cli/Launcher.groovy
@@ -29,6 +29,7 @@ import groovy.transform.CompileDynamic
 import groovy.transform.CompileStatic
 import groovy.transform.PackageScope
 import groovy.util.logging.Slf4j
+import nextflow.BuildInfo
 import nextflow.exception.AbortOperationException
 import nextflow.exception.AbortRunException
 import nextflow.exception.ConfigParseException
@@ -685,10 +686,25 @@ class Launcher {
             SPLASH
         }
         else {
-            "${APP_NAME} version ${APP_VER}.${APP_BUILDNUM}"
+            "${APP_NAME} version ${BuildInfo.version}.${BuildInfo.buildNum}"
         }
 
     }
 
+    /*
+     * The application 'logo'
+     */
+    /*
+     * The application 'logo'
+     */
+    static public final String SPLASH =
+
+"""
+      N E X T F L O W
+      version ${BuildInfo.version} build ${BuildInfo.buildNum}
+      created ${BuildInfo.timestampUTC} ${BuildInfo.timestampDelta}
+      cite doi:10.1038/nbt.3820
+      http://nextflow.io
+"""
 
 }
diff --git a/modules/nextflow/src/main/groovy/nextflow/container/ContainerConfig.groovy b/modules/nextflow/src/main/groovy/nextflow/container/ContainerConfig.groovy
index 78e8caef71..71fb346713 100644
--- a/modules/nextflow/src/main/groovy/nextflow/container/ContainerConfig.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/container/ContainerConfig.groovy
@@ -93,6 +93,8 @@ class ContainerConfig extends LinkedHashMap {
             return null
         if( eng=='docker' || eng=='podman' )
             return '--rm --privileged'
+        if( singularityOciMode() )
+            return '-B /dev/fuse'
         if( eng=='singularity' || eng=='apptainer' )
             return null
         log.warn "Fusion file system is not supported by '$eng' container engine"
diff --git a/modules/nextflow/src/main/groovy/nextflow/container/SingularityCache.groovy b/modules/nextflow/src/main/groovy/nextflow/container/SingularityCache.groovy
index 7e2134aa67..2fa7c85db4 100644
--- a/modules/nextflow/src/main/groovy/nextflow/container/SingularityCache.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/container/SingularityCache.groovy
@@ -99,7 +99,7 @@ class SingularityCache {
     }
 
     /**
-     * Create the specified directory if not exists
+     * Create the specified directory if it does not exist
      *
      * @param
      *      str A path string representing a folder where store the singularity images once downloaded
diff --git a/modules/nextflow/src/main/groovy/nextflow/container/resolver/DefaultContainerResolver.groovy b/modules/nextflow/src/main/groovy/nextflow/container/resolver/DefaultContainerResolver.groovy
index 54432faefa..34cafe65b5 100644
--- a/modules/nextflow/src/main/groovy/nextflow/container/resolver/DefaultContainerResolver.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/container/resolver/DefaultContainerResolver.groovy
@@ -39,10 +39,20 @@ class DefaultContainerResolver implements ContainerResolver {
             return ContainerInfo.EMPTY
         }
 
+        final ret = resolveImage0(task, imageName)
+        return new ContainerInfo(imageName, ret, ret)
+    }
+
+    private String resolveImage0(TaskRun task, String imageName) {
         final cfg = task.getContainerConfig()
         final handler = new ContainerHandler(cfg)
-        final ret = handler.normalizeImageName(imageName)
-        return new ContainerInfo(imageName, ret, ret)
+        return handler.normalizeImageName(imageName)
+    }
+
+    ContainerInfo resolveImage(TaskRun task, String imageName, String hashKey) {
+        assert imageName
+        final ret = resolveImage0(task, imageName)
+        return new ContainerInfo(imageName, ret, hashKey)
     }
 
 }
diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/GridTaskHandler.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/GridTaskHandler.groovy
index 143703b551..3ad6430856 100644
--- a/modules/nextflow/src/main/groovy/nextflow/executor/GridTaskHandler.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/executor/GridTaskHandler.groovy
@@ -478,7 +478,7 @@ class GridTaskHandler extends TaskHandler implements FusionAwareTask {
                 return true
             }
             // if the task is not complete (ie submitted or running)
-            // AND the work-dir does not exists ==> something is wrong
+            // AND the work-dir does not exist ==> something is wrong
             task.error = new ProcessException("Task work directory is missing (!)")
             // sanity check does not pass
             return false
diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/CH.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/CH.groovy
index ba8727808e..6069317f86 100644
--- a/modules/nextflow/src/main/groovy/nextflow/extension/CH.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/extension/CH.groovy
@@ -1,5 +1,7 @@
 package nextflow.extension
 
+import static nextflow.Channel.*
+
 import groovy.transform.CompileStatic
 import groovy.transform.PackageScope
 import groovy.util.logging.Slf4j
@@ -15,8 +17,6 @@ import nextflow.Channel
 import nextflow.Global
 import nextflow.NF
 import nextflow.Session
-import static nextflow.Channel.STOP
-
 /**
  * Helper class to handle channel internal api ops
  *
@@ -30,7 +30,14 @@ class CH {
         return (Session) Global.session
     }
 
-    static private Map bridges = new HashMap<>(10)
+    static class Topic {
+        DataflowBroadcast broadcaster = new DataflowBroadcast()
+        List sources = new ArrayList<>(10)
+    }
+
+    static final private Map allTopics = new HashMap<>(10)
+
+    static final private Map bridges = new HashMap<>(10)
 
     static DataflowReadChannel getReadChannel(channel) {
         if (channel instanceof DataflowQueue)
@@ -66,7 +73,15 @@ class CH {
     }
 
     static void broadcast() {
-        // connect all dataflow queue variables to associated broadcast channel 
+        // connect all broadcast topics, note this must be before the following
+        // "bridging" step because it can modify the final network topology
+        connectTopics()
+        // bridge together all broadcast channels
+        bridgeChannels()
+    }
+
+    static private void bridgeChannels() {
+        // connect all dataflow queue variables to associated broadcast channel
         for( DataflowQueue queue : bridges.keySet() ) {
             log.trace "Bridging dataflow queue=$queue"
             def broadcast = bridges.get(queue)
@@ -74,6 +89,25 @@ class CH {
         }
     }
 
+    static private void connectTopics() {
+        for( Topic topic : allTopics.values() ) {
+            if( topic.sources ) {
+                // get the list of source channels for this topic
+                final ch = new ArrayList(topic.sources)
+                // the mix operator requires at least two sources, add an empty channel if needed
+                if( ch.size()==1 )
+                    ch.add(empty())
+                // map write channels to read channels
+                final sources = ch.collect(it -> getReadChannel(it))
+                // mix all of them 
+                new MixOp(sources).withTarget(topic.broadcaster).apply()
+            }
+            else {
+                topic.broadcaster.bind(STOP)
+            }
+        }
+    }
+
     static void init() { bridges.clear() }
 
     @PackageScope
@@ -102,6 +136,31 @@ class CH {
         return new DataflowQueue()
     }
 
+    static DataflowBroadcast topic(String name) {
+        synchronized (allTopics) {
+            def topic = allTopics[name]
+            if( topic!=null )
+                return topic.broadcaster
+            // create a new topic
+            topic = new Topic()
+            allTopics[name] = topic
+            return topic.broadcaster
+        }
+    }
+
+    static DataflowWriteChannel createTopicSource(String name) {
+        synchronized (allTopics) {
+            def topic = allTopics.get(name)
+            if( topic==null ) {
+                topic = new Topic()
+                allTopics.put(name, topic)
+            }
+            final result = CH.create()
+            topic.sources.add(result)
+            return result
+        }
+    }
+
     static boolean isChannel(obj) {
         obj instanceof DataflowReadChannel || obj instanceof DataflowWriteChannel
     }
diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/GroupTupleOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/GroupTupleOp.groovy
index 36e8d2f1a8..e51a3b6bc7 100644
--- a/modules/nextflow/src/main/groovy/nextflow/extension/GroupTupleOp.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/extension/GroupTupleOp.groovy
@@ -96,7 +96,7 @@ class GroupTupleOp {
         final len = tuple.size()
 
         final List items = groups.getOrCreate(key) {    // get the group for the specified key
-            def result = new ArrayList(len)             // create if does not exists
+            def result = new ArrayList(len)             // create if does not exist
             for( int i=0; i others
+    private DataflowWriteChannel target
 
     MixOp(DataflowReadChannel source, DataflowReadChannel other) {
         this.source = source
@@ -47,8 +48,21 @@ class MixOp {
         this.others = others.toList()
     }
 
+    MixOp(List channels) {
+        if( channels.size()<2 )
+            throw new IllegalArgumentException("Mix operator requires at least 2 source channels")
+        this.source = channels.get(0)
+        this.others = channels.subList(1, channels.size())
+    }
+
+    MixOp withTarget(DataflowWriteChannel target) {
+        this.target = target
+        return this
+    }
+
     DataflowWriteChannel apply() {
-        def target = CH.create()
+        if( target == null )
+            target = CH.create()
         def count = new AtomicInteger( others.size()+1 )
         def handlers = [
                 onNext: { target << it },
diff --git a/modules/nextflow/src/main/groovy/nextflow/file/FileCollector.groovy b/modules/nextflow/src/main/groovy/nextflow/file/FileCollector.groovy
index 4135e8b2d7..de20b47cd3 100644
--- a/modules/nextflow/src/main/groovy/nextflow/file/FileCollector.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/file/FileCollector.groovy
@@ -217,7 +217,7 @@ abstract class FileCollector implements Closeable {
      * Save the entries collected grouping them into files whose name is given by the
      * correspondent group key
      *
-     * @param target A {@link Path} to the folder where files will be saved. If the folder does not exists, is created
+     * @param target A {@link Path} to the folder where files will be saved. If the folder does not exist, is created
      *              automatically
      * @return The list of files where entries have been saved.
      */
diff --git a/modules/nextflow/src/main/groovy/nextflow/fusion/FusionConfig.groovy b/modules/nextflow/src/main/groovy/nextflow/fusion/FusionConfig.groovy
index e6b71f0dd0..e67006cb14 100644
--- a/modules/nextflow/src/main/groovy/nextflow/fusion/FusionConfig.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/fusion/FusionConfig.groovy
@@ -21,6 +21,8 @@ import groovy.transform.CompileStatic
 import groovy.transform.Memoized
 import nextflow.Global
 import nextflow.SysEnv
+import nextflow.util.MemoryUnit
+
 /**
  * Model Fusion config options
  *
@@ -44,6 +46,7 @@ class FusionConfig {
     final private boolean tagsEnabled
     final private String tagsPattern
     final private boolean privileged
+    final private MemoryUnit cacheSize
 
     boolean enabled() { enabled }
 
@@ -63,6 +66,8 @@ class FusionConfig {
 
     String tagsPattern() { tagsPattern }
 
+    MemoryUnit cacheSize() { cacheSize }
+
     URL containerConfigUrl() {
         this.containerConfigUrl ? new URL(this.containerConfigUrl) : null
     }
@@ -81,6 +86,7 @@ class FusionConfig {
         this.tagsEnabled = opts.tags==null || opts.tags.toString()!='false'
         this.tagsPattern = (opts.tags==null || (opts.tags instanceof Boolean && opts.tags)) ? DEFAULT_TAGS : ( opts.tags !instanceof Boolean ? opts.tags as String : null )
         this.privileged = opts.privileged==null || opts.privileged.toString()=='true'
+        this.cacheSize = opts.cacheSize as MemoryUnit
         if( containerConfigUrl && !validProtocol(containerConfigUrl))
             throw new IllegalArgumentException("Fusion container config URL should start with 'http:' or 'https:' protocol prefix - offending value: $containerConfigUrl")
     }
diff --git a/modules/nextflow/src/main/groovy/nextflow/fusion/FusionEnvProvider.groovy b/modules/nextflow/src/main/groovy/nextflow/fusion/FusionEnvProvider.groovy
index a1c2f6e7e3..b31e11e5d9 100644
--- a/modules/nextflow/src/main/groovy/nextflow/fusion/FusionEnvProvider.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/fusion/FusionEnvProvider.groovy
@@ -42,6 +42,8 @@ class FusionEnvProvider {
             result.FUSION_LOG_OUTPUT = config.logOutput()
         if( config.logLevel() )
             result.FUSION_LOG_LEVEL = config.logLevel()
+        if( config.cacheSize() )
+            result.FUSION_CACHE_SIZE = "${config.cacheSize().toMega()}M"
         return result
     }
 }
diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy
index 490f5225e8..1750f6f293 100644
--- a/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy
@@ -25,10 +25,12 @@ import java.nio.file.NoSuchFileException
 import java.nio.file.Path
 import java.nio.file.PathMatcher
 import java.util.concurrent.ExecutorService
+import java.util.regex.Pattern
 
 import groovy.transform.CompileDynamic
 import groovy.transform.CompileStatic
 import groovy.transform.EqualsAndHashCode
+import groovy.transform.Memoized
 import groovy.transform.PackageScope
 import groovy.transform.ToString
 import groovy.util.logging.Slf4j
@@ -38,6 +40,7 @@ import nextflow.Session
 import nextflow.extension.FilesEx
 import nextflow.file.FileHelper
 import nextflow.file.TagAwareFile
+import nextflow.fusion.FusionHelper
 import nextflow.util.PathTrie
 /**
  * Implements the {@code publishDir} directory. It create links or copies the output
@@ -51,10 +54,14 @@ import nextflow.util.PathTrie
 @CompileStatic
 class PublishDir {
 
+    final static private Pattern FUSION_PATH_REGEX = ~/^\/fusion\/([^\/]+)\/(.*)/
+
     enum Mode { SYMLINK, LINK, COPY, MOVE, COPY_NO_FOLLOW, RELLINK }
 
     private Map makeCache = new HashMap<>()
 
+    private Session session = Global.session as Session
+
     /**
      * The target path where create the links or copy the output files
      */
@@ -117,10 +124,18 @@ class PublishDir {
 
     private boolean nullPathWarn
 
-    private String taskName
+    private TaskRun task
 
     @Lazy
-    private ExecutorService threadPool = { def sess = Global.session as Session; sess.publishDirExecutorService() }()
+    private ExecutorService threadPool = { session.publishDirExecutorService() }()
+
+    protected String getTaskName() {
+        return task?.getName()
+    }
+
+    protected Map getTaskInputs() {
+        return task?.getInputFilesMap()
+    }
 
     void setPath( def value ) {
         final resolved = value instanceof Closure ? value.call() : value
@@ -202,7 +217,6 @@ class PublishDir {
         return result
     }
 
-    @CompileStatic
     protected void apply0(Set files) {
         assert path
 
@@ -280,13 +294,10 @@ class PublishDir {
         this.sourceDir = task.targetDir
         this.sourceFileSystem = sourceDir.fileSystem
         this.stageInMode = task.config.stageInMode
-        this.taskName = task.name
 
         apply0(task.outputFiles)
     }
 
-
-    @CompileStatic
     protected void apply1(Path source, boolean inProcess ) {
 
         def target = sourceDir ? sourceDir.relativize(source) : source.getFileName()
@@ -327,7 +338,6 @@ class PublishDir {
 
     }
 
-    @CompileStatic
     protected Path resolveDestination(target) {
 
         if( target instanceof Path ) {
@@ -346,7 +356,6 @@ class PublishDir {
         throw new IllegalArgumentException("Not a valid publish target path: `$target` [${target?.class?.name}]")
     }
 
-    @CompileStatic
     protected void safeProcessFile(Path source, Path target) {
         try {
             processFile(source, target)
@@ -354,15 +363,20 @@ class PublishDir {
         catch( Throwable e ) {
             log.warn "Failed to publish file: ${source.toUriString()}; to: ${target.toUriString()} [${mode.toString().toLowerCase()}] -- See log file for details", e
             if( NF.strictMode || failOnError){
-                final session = Global.session as Session
                 session?.abort(e)
             }
         }
     }
 
-    @CompileStatic
     protected void processFile( Path source, Path destination ) {
 
+        // resolve Fusion symlink if applicable
+        if( FusionHelper.isFusionEnabled(session) ) {
+            final inputs = getTaskInputs()
+            if( source.name in inputs )
+                source = resolveFusionLink(inputs[source.name])
+        }
+
         // create target dirs if required
         makeDirs(destination.parent)
 
@@ -386,6 +400,29 @@ class PublishDir {
         notifyFilePublish(destination, source)
     }
 
+    /**
+     * Resolve a Fusion symlink by following the .fusion.symlinks
+     * file in the task directory until the original file is reached.
+     *
+     * @param file
+     */
+    protected Path resolveFusionLink(Path file) {
+        while( file.name in getFusionLinks(file.parent) )
+            file = file.text.replaceFirst(FUSION_PATH_REGEX) { _, scheme, path -> "${scheme}://${path}" } as Path
+        return file
+    }
+
+    @Memoized
+    protected List getFusionLinks(Path workDir) {
+        try {
+            final file = workDir.resolve('.fusion.symlinks')
+            return file.text.tokenize('\n')
+        }
+        catch( NoSuchFileException e ) {
+            return List.of()
+        }
+    }
+
     private String real0(Path p) {
         try {
             // resolve symlink if it's file in the default (posix) file system
@@ -421,8 +458,9 @@ class PublishDir {
         final s1 = real0(sourceDir)
         if( t1.startsWith(s1) ) {
             def msg = "Refusing to publish file since destination path conflicts with the task work directory!"
-            if( taskName )
-                msg += "\n- offending task  : $taskName"
+            def name0 = getTaskName()
+            if( name0 )
+                msg += "\n- offending task  : $name0"
             msg += "\n- offending file  : $target"
             if( t1 != target.toString() )
                 msg += "\n- real destination: $t1"
@@ -437,7 +475,6 @@ class PublishDir {
         return !mode || mode == Mode.SYMLINK || mode == Mode.RELLINK
     }
 
-    @CompileStatic
     protected void processFileImpl( Path source, Path destination ) {
         log.trace "publishing file: $source -[$mode]-> $destination"
 
@@ -465,13 +502,11 @@ class PublishDir {
         }
     }
 
-    @CompileStatic
-    private void createPublishDir() {
+    protected void createPublishDir() {
         makeDirs(this.path)
     }
 
-    @CompileStatic
-    private void makeDirs(Path dir) {
+    protected void makeDirs(Path dir) {
         if( !dir || makeCache.containsKey(dir) )
             return
 
@@ -490,7 +525,6 @@ class PublishDir {
      * That valid publish mode has been selected
      * Note: link and symlinks are not allowed across different file system
      */
-    @CompileStatic
     @PackageScope
     void validatePublishMode() {
         if( log.isTraceEnabled() )
@@ -511,11 +545,7 @@ class PublishDir {
     }
 
     protected void notifyFilePublish(Path destination, Path source=null) {
-        final sess = Global.session
-        if (sess instanceof Session) {
-            sess.notifyFilePublish(destination, source)
-        }
+        session.notifyFilePublish(destination, source)
     }
 
-
 }
diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy
index 99204ba94a..08f08546de 100644
--- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy
@@ -645,7 +645,7 @@ class TaskProcessor {
         }
 
         if( !task.config.getStoreDir().exists() ) {
-            log.trace "[${safeTaskName(task)}] Store dir does not exists > ${task.config.storeDir} -- return false"
+            log.trace "[${safeTaskName(task)}] Store dir does not exist > ${task.config.storeDir} -- return false"
             // no folder -> no cached result
             return false
         }
diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy
index 628f7c7e19..8d9d7aaa86 100644
--- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy
@@ -404,7 +404,7 @@ class TaskRun implements Cloneable {
      * @return A map object containing all the task input files as  pairs
      */
     Map getInputFilesMap() {
-        def result = [:]
+        final result = [:]
         for( FileHolder it : inputFiles )
             result.put(it.stageName, it.storePath)
         return result
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy
index 035588695f..1343631cab 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy
@@ -184,9 +184,15 @@ class ProcessDef extends BindableDef implements IterableDef, ChainableDef {
         if( feedbackChannels && feedbackChannels.size() != declaredOutputs.size() )
             throw new ScriptRuntimeException("Process `$processName` inputs and outputs do not have the same cardinality - Feedback loop is not supported"  )
 
-        for(int i=0; i entry : opts )
+            setProperty(entry.key, entry.value)
+    }
+
+    void setName(String name) {
+        if( !ConfigHelper.isValidIdentifier(name) ) {
+            final msg = "Output name '$name' is not valid -- Make sure it starts with an alphabetic or underscore character and it does not contain any blank, dot or other special characters"
+            throw new IllegalArgumentException(msg)
+        }
+        this.name = name
     }
 
     String getName() {
         return name
     }
 
+    void setTopic(String topic) {
+        if( !ConfigHelper.isValidIdentifier(topic) ) {
+            final msg = "Output topic '$topic' is not valid -- Make sure it starts with an alphabetic or underscore character and it does not contain any blank, dot or other special characters"
+            throw new IllegalArgumentException(msg)
+        }
+        this.topic = topic
+    }
+
+    String getTopic() {
+        return topic
+    }
+
     void setChannel(DataflowWriteChannel channel) {
         this.channel = channel
     }
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ScriptBinding.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ScriptBinding.groovy
index 09293c144b..a95bbd5328 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ScriptBinding.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ScriptBinding.groovy
@@ -239,7 +239,11 @@ class ScriptBinding extends WorkflowBinding {
         }
 
         private ParamsMap allowNames(Set names) {
-            readOnlyNames.removeAll(names)
+            for( String name : names ) {
+                final name2 = name.contains('-') ? hyphenToCamelCase(name) : camelCaseToHyphen(name)
+                readOnlyNames.remove(name)
+                readOnlyNames.remove(name2)
+            }
             return this
         }
 
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/WorkflowMetadata.groovy b/modules/nextflow/src/main/groovy/nextflow/script/WorkflowMetadata.groovy
index af00db5be0..745fa7e6a4 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/WorkflowMetadata.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/WorkflowMetadata.groovy
@@ -463,10 +463,10 @@ class WorkflowMetadata {
      */
     protected void safeMailNotification() {
         try {
-            def notifier = new WorkflowNotifier()
-            notifier.workflow = this
-            notifier.config = session.config
-            notifier.variables = NF.binding.variables
+            final notifier = new WorkflowNotifier(
+                workflow: this,
+                config: session.config,
+                variables: NF.binding.variables )
             notifier.sendNotification()
         }
         catch (Exception e) {
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/WorkflowNotifier.groovy b/modules/nextflow/src/main/groovy/nextflow/script/WorkflowNotifier.groovy
index 7cc351048f..13228a2b42 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/WorkflowNotifier.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/WorkflowNotifier.groovy
@@ -110,9 +110,9 @@ class WorkflowNotifier {
         List templates = []
         normaliseTemplate0(notification.template, templates)
         if (templates) {
-            final binding = normaliseBindings0(notification.binding)
+            final binding = normaliseBindings0(notification.attributes)
 
-            templates.each { file ->
+            for( File file : templates ) {
                 def content = loadMailTemplate(file, binding)
                 def plain = file.extension == 'txt'
                 if (plain) {
@@ -135,7 +135,7 @@ class WorkflowNotifier {
     protected Map normaliseBindings0(binding) {
 
         if (binding == null)
-            return null
+            return Map.of()
 
         if (binding instanceof Map)
             return binding
@@ -218,8 +218,10 @@ class WorkflowNotifier {
 
     private String loadMailTemplate0(InputStream source, Map binding) {
         def map = new HashMap()
-        map.putAll(variables)
-        map.putAll(binding)
+        if( variables )
+            map.putAll(variables)
+        if( binding )
+            map.putAll(binding)
 
         def template = new GStringTemplateEngine().createTemplate(new InputStreamReader(source))
         template.make(map).toString()
diff --git a/modules/nextflow/src/main/resources/META-INF/build-info.properties b/modules/nextflow/src/main/resources/META-INF/build-info.properties
new file mode 100644
index 0000000000..95836440d9
--- /dev/null
+++ b/modules/nextflow/src/main/resources/META-INF/build-info.properties
@@ -0,0 +1,4 @@
+build=5900
+version=23.11.0-edge
+timestamp=1702494578227
+commitId=887f06f4d
diff --git a/modules/nextflow/src/main/resources/META-INF/plugins-info.txt b/modules/nextflow/src/main/resources/META-INF/plugins-info.txt
index 4f2d453db3..74e02769f1 100644
--- a/modules/nextflow/src/main/resources/META-INF/plugins-info.txt
+++ b/modules/nextflow/src/main/resources/META-INF/plugins-info.txt
@@ -1,9 +1,9 @@
-nf-amazon@2.1.4
-nf-azure@1.3.2
-nf-cloudcache@0.3.0
-nf-codecommit@0.1.5
-nf-console@1.0.6
-nf-ga4gh@1.1.0
-nf-google@1.8.3
-nf-tower@1.6.3
-nf-wave@1.0.0
\ No newline at end of file
+nf-amazon@2.2.0
+nf-azure@1.4.0
+nf-cloudcache@0.3.1
+nf-codecommit@0.1.6
+nf-console@1.0.7
+nf-ga4gh@1.1.1
+nf-google@1.9.0
+nf-tower@1.7.0
+nf-wave@1.1.0
\ No newline at end of file
diff --git a/modules/nextflow/src/test/groovy/nextflow/NextflowMetaTest.groovy b/modules/nextflow/src/test/groovy/nextflow/NextflowMetaTest.groovy
index 0abe709dc9..5abaac2ba9 100644
--- a/modules/nextflow/src/test/groovy/nextflow/NextflowMetaTest.groovy
+++ b/modules/nextflow/src/test/groovy/nextflow/NextflowMetaTest.groovy
@@ -1,13 +1,11 @@
 package nextflow
 
-import spock.lang.Specification
+import static nextflow.extension.Bolts.*
 
 import java.text.SimpleDateFormat
 
-import static nextflow.extension.Bolts.DATETIME_FORMAT
-
+import spock.lang.Specification
 import spock.lang.Unroll
-
 /**
  *
  * @author Paolo Di Tommaso 
@@ -23,7 +21,7 @@ class NextflowMetaTest extends Specification {
 
     def 'should convert to map'() {
         given:
-        def meta = new NextflowMeta('10.12.0', 123, Const.APP_TIMESTAMP_UTC)
+        def meta = new NextflowMeta('10.12.0', 123, BuildInfo.timestampUTC)
         meta.enableDsl2()
 
         when:
@@ -32,7 +30,7 @@ class NextflowMetaTest extends Specification {
         map.version == '10.12.0'
         map.build == 123
         map.enable.dsl == 2
-        dateToString((Date) map.timestamp) == Const.APP_TIMESTAMP_UTC
+        dateToString((Date) map.timestamp) == BuildInfo.timestampUTC
 
     }
 
diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/CmdCloneTest.groovy b/modules/nextflow/src/test/groovy/nextflow/cli/CmdCloneTest.groovy
index 5ecbcda749..a65e1cb1e6 100644
--- a/modules/nextflow/src/test/groovy/nextflow/cli/CmdCloneTest.groovy
+++ b/modules/nextflow/src/test/groovy/nextflow/cli/CmdCloneTest.groovy
@@ -18,6 +18,7 @@ package nextflow.cli
 import java.nio.file.Files
 
 import nextflow.plugin.Plugins
+import spock.lang.IgnoreIf
 import spock.lang.Requires
 import spock.lang.Specification
 /**
@@ -26,6 +27,7 @@ import spock.lang.Specification
  */
 class CmdCloneTest extends Specification {
 
+    @IgnoreIf({System.getenv('NXF_SMOKE')})
     @Requires({System.getenv('NXF_GITHUB_ACCESS_TOKEN')})
     def testClone() {
 
diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/CmdRunTest.groovy b/modules/nextflow/src/test/groovy/nextflow/cli/CmdRunTest.groovy
index e18b401a76..004128cc22 100644
--- a/modules/nextflow/src/test/groovy/nextflow/cli/CmdRunTest.groovy
+++ b/modules/nextflow/src/test/groovy/nextflow/cli/CmdRunTest.groovy
@@ -172,7 +172,7 @@ class CmdRunTest extends Specification {
         cmd.parsedParams()
         then:
         def e = thrown(AbortOperationException)
-        e.message == 'Specified params file does not exists: /missing/path.yml'
+        e.message == 'Specified params file does not exist: /missing/path.yml'
 
         cleanup:
         folder?.delete()
diff --git a/modules/nextflow/src/test/groovy/nextflow/container/ContainerConfigTest.groovy b/modules/nextflow/src/test/groovy/nextflow/container/ContainerConfigTest.groovy
index 99ed6439fe..29ce5704e4 100644
--- a/modules/nextflow/src/test/groovy/nextflow/container/ContainerConfigTest.groovy
+++ b/modules/nextflow/src/test/groovy/nextflow/container/ContainerConfigTest.groovy
@@ -95,6 +95,9 @@ class ContainerConfigTest extends Specification {
         [engine:'docker']                               | '--rm --privileged'
         [engine:'podman']                               | '--rm --privileged'
         and:
+        [engine: 'singularity']                         | null
+        [engine: 'singularity', oci:true]               | '-B /dev/fuse'
+        and:
         [engine:'docker', fusionOptions:'--cap-add foo']| '--cap-add foo'
         [engine:'podman', fusionOptions:'--cap-add bar']| '--cap-add bar'
         and:
diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/PublishOpTest.groovy b/modules/nextflow/src/test/groovy/nextflow/extension/PublishOpTest.groovy
index b968b284c7..e904387db3 100644
--- a/modules/nextflow/src/test/groovy/nextflow/extension/PublishOpTest.groovy
+++ b/modules/nextflow/src/test/groovy/nextflow/extension/PublishOpTest.groovy
@@ -42,6 +42,7 @@ class PublishOpTest extends BaseSpec {
         def BASE = folder
         def sess = Mock(Session) {
             getWorkDir() >> BASE
+            getConfig() >> [:]
         }
         Global.session = sess
 
diff --git a/modules/nextflow/src/test/groovy/nextflow/fusion/FusionConfigTest.groovy b/modules/nextflow/src/test/groovy/nextflow/fusion/FusionConfigTest.groovy
index b6d176038f..d5e511d365 100644
--- a/modules/nextflow/src/test/groovy/nextflow/fusion/FusionConfigTest.groovy
+++ b/modules/nextflow/src/test/groovy/nextflow/fusion/FusionConfigTest.groovy
@@ -17,7 +17,7 @@
 
 package nextflow.fusion
 
-
+import nextflow.util.MemoryUnit
 import spock.lang.Specification
 import spock.lang.Unroll
 /**
@@ -85,6 +85,21 @@ class FusionConfigTest extends Specification {
         [logOutput: 'stdout']           | null      | 'stdout'
     }
 
+    def 'should configure cache size' () {
+        given:
+        def opts = new FusionConfig(OPTS)
+        expect:
+        opts.cacheSize() == SIZE
+
+        where:
+        OPTS                            | SIZE
+        [:]                             | null
+        [cacheSize: 100]                | MemoryUnit.of(100)
+        [cacheSize: '100']              | MemoryUnit.of(100)
+        [cacheSize: '100.MB']           | MemoryUnit.of('100.MB')
+    }
+
+
     @Unroll
     def 'should configure tags' () {
         given:
diff --git a/modules/nextflow/src/test/groovy/nextflow/processor/PublishDirTest.groovy b/modules/nextflow/src/test/groovy/nextflow/processor/PublishDirTest.groovy
index 3c858344cb..613fead059 100644
--- a/modules/nextflow/src/test/groovy/nextflow/processor/PublishDirTest.groovy
+++ b/modules/nextflow/src/test/groovy/nextflow/processor/PublishDirTest.groovy
@@ -20,6 +20,7 @@ import java.nio.file.FileSystems
 import java.nio.file.Files
 import java.nio.file.Paths
 
+import nextflow.Global
 import nextflow.Session
 import spock.lang.Specification
 import test.TestHelper
@@ -114,8 +115,9 @@ class PublishDirTest extends Specification {
     }
 
     def 'should create symlinks for output files' () {
-
         given:
+        Global.session = Mock(Session) { getConfig()>>[:] }
+        and:
         def folder = Files.createTempDirectory('nxf')
         folder.resolve('work-dir').mkdir()
         folder.resolve('work-dir/file1.txt').text = 'aaa'
diff --git a/modules/nextflow/src/test/groovy/nextflow/scm/GiteaRepositoryProviderTest.groovy b/modules/nextflow/src/test/groovy/nextflow/scm/GiteaRepositoryProviderTest.groovy
index f933b469d2..bdfea1d0f8 100644
--- a/modules/nextflow/src/test/groovy/nextflow/scm/GiteaRepositoryProviderTest.groovy
+++ b/modules/nextflow/src/test/groovy/nextflow/scm/GiteaRepositoryProviderTest.groovy
@@ -16,6 +16,7 @@
 
 package nextflow.scm
 
+import spock.lang.IgnoreIf
 import spock.lang.Requires
 import spock.lang.Specification
 
@@ -76,6 +77,7 @@ class GiteaRepositoryProviderTest extends Specification {
 
     }
 
+    @IgnoreIf({System.getenv('NXF_SMOKE')})
     @Requires({System.getenv('NXF_GITEA_ACCESS_TOKEN')})
     def 'should read file content'() {
 
diff --git a/modules/nextflow/src/test/groovy/nextflow/script/ScriptBindingTest.groovy b/modules/nextflow/src/test/groovy/nextflow/script/ScriptBindingTest.groovy
index 57089deafb..c86b2a36dd 100644
--- a/modules/nextflow/src/test/groovy/nextflow/script/ScriptBindingTest.groovy
+++ b/modules/nextflow/src/test/groovy/nextflow/script/ScriptBindingTest.groovy
@@ -144,23 +144,31 @@ class ScriptBindingTest extends Specification {
         def map = new ScriptBinding.ParamsMap()
         map['alpha'] = 0
         map['alpha'] = 1
+        map['alphaBeta'] = 0
+        map['alphaBeta'] = 1
         map['delta'] = 2
         map['gamma'] = 3
         then:
         map.alpha == 0
+        map.alphaBeta == 0
+        map.'alpha-beta' == 0
         map.delta == 2
         map.gamma == 3
 
         when:
-        def copy = map.copyWith(foo:1, omega: 9)
+        def copy = map.copyWith(foo:1, alphaBeta: 4, omega: 9)
         then:
         copy.foo == 1
+        copy.alphaBeta == 4
+        copy.'alpha-beta' == 4
         copy.delta == 2
         copy.gamma == 3
         copy.omega == 9
         and:
         // source does not change
         map.alpha == 0
+        map.alphaBeta == 0
+        map.'alpha-beta' == 0
         map.delta == 2
         map.gamma == 3
         !map.containsKey('omega')
diff --git a/modules/nextflow/src/test/groovy/nextflow/script/WorkflowMetadataTest.groovy b/modules/nextflow/src/test/groovy/nextflow/script/WorkflowMetadataTest.groovy
index 05e370c612..4d05157297 100644
--- a/modules/nextflow/src/test/groovy/nextflow/script/WorkflowMetadataTest.groovy
+++ b/modules/nextflow/src/test/groovy/nextflow/script/WorkflowMetadataTest.groovy
@@ -16,12 +16,11 @@
 
 package nextflow.script
 
-
 import java.nio.file.Paths
 import java.time.OffsetDateTime
 
-import nextflow.Const
 import nextflow.Session
+import nextflow.BuildInfo
 import nextflow.exception.WorkflowScriptErrorException
 import nextflow.trace.TraceRecord
 import nextflow.trace.WorkflowStats
@@ -83,9 +82,9 @@ class WorkflowMetadataTest extends Specification {
         metadata.start <= OffsetDateTime.now()
         metadata.complete == null
         metadata.commandLine == 'nextflow run -this -that'
-        metadata.nextflow.version == new VersionNumber(Const.APP_VER)
-        metadata.nextflow.build == Const.APP_BUILDNUM
-        metadata.nextflow.timestamp == Const.APP_TIMESTAMP_UTC
+        metadata.nextflow.version == new VersionNumber(BuildInfo.version)
+        metadata.nextflow.build == BuildInfo.buildNum as int
+        metadata.nextflow.timestamp == BuildInfo.timestampUTC
         metadata.profile == 'standard'
         metadata.sessionId == session.uniqueId
         metadata.runName == session.runName
diff --git a/modules/nextflow/src/test/groovy/nextflow/script/WorkflowNotifierTest.groovy b/modules/nextflow/src/test/groovy/nextflow/script/WorkflowNotifierTest.groovy
index 91bfeb0bc2..e41a4e31cf 100644
--- a/modules/nextflow/src/test/groovy/nextflow/script/WorkflowNotifierTest.groovy
+++ b/modules/nextflow/src/test/groovy/nextflow/script/WorkflowNotifierTest.groovy
@@ -16,6 +16,7 @@
 
 package nextflow.script
 
+import java.nio.file.Files
 import java.nio.file.Paths
 import java.time.Instant
 import java.time.OffsetDateTime
@@ -256,7 +257,7 @@ class WorkflowNotifierTest extends Specification {
         when:
         workflow.success = false
         workflow.runName = 'bar'
-        mail = notifier.createMail([to:'alpha@dot.com', from:'beta@dot.com', template: ['/some/file.txt', '/other/file.html'], binding: [one:1, two:2]])
+        mail = notifier.createMail([to:'alpha@dot.com', from:'beta@dot.com', template: ['/some/file.txt', '/other/file.html'], attributes: [one:1, two:2]])
         then:
         1 * notifier.loadMailTemplate(new File('/some/file.txt'), [one:1, two:2]) >> 'TEXT template'
         1 * notifier.loadMailTemplate(new File('/other/file.html'), [one:1, two:2]) >> 'HTML template'
@@ -268,6 +269,40 @@ class WorkflowNotifierTest extends Specification {
 
     }
 
+    def 'should create notification mail with custom template' () {
+        given:
+        def folder  = Files.createTempDirectory('test')
+        def template = folder.resolve('template.txt').toFile(); template.text = 'Hello world!'
+
+        and:
+        Mail mail
+        def workflow = new WorkflowMetadata()
+        def notifier = Spy(WorkflowNotifier)
+        notifier.@workflow = workflow
+
+        /*
+         * create success completion  *default* notification email
+         */
+        when:
+        workflow.success = true
+        workflow.runName = 'foo'
+        mail = notifier.createMail([to:'paolo@yo.com', from:'bot@nextflow.com', template: template])
+        then:
+        0 * notifier.loadDefaultTextTemplate()
+        0 * notifier.loadDefaultHtmlTemplate()
+        1 * notifier.loadMailTemplate(template, [:])
+        and:
+        mail.to == 'paolo@yo.com'
+        mail.from == 'bot@nextflow.com'
+        mail.subject == 'Workflow completion [foo] - SUCCEED'
+        mail.text == 'Hello world!'
+        !mail.body 
+        !mail.attachments
+
+        cleanup:
+        folder?.deleteDir()
+    }
+
 
     def 'should send notification email' () {
 
diff --git a/modules/nextflow/src/test/groovy/nextflow/script/params/ParamsOutTest.groovy b/modules/nextflow/src/test/groovy/nextflow/script/params/ParamsOutTest.groovy
index a3225d3f05..156bca6a85 100644
--- a/modules/nextflow/src/test/groovy/nextflow/script/params/ParamsOutTest.groovy
+++ b/modules/nextflow/src/test/groovy/nextflow/script/params/ParamsOutTest.groovy
@@ -812,7 +812,7 @@ class ParamsOutTest extends Dsl2Spec {
 
             process hola {
               output:
-              path "${x}_name", emit: aaa
+              path "${x}_name", emit: aaa, topic: 'foo'
               path "${x}_${y}.fa", emit: bbb 
               path "simple.txt", emit: ccc 
               path "data/sub/dir/file:${x}.fa", emit: ddd 
@@ -841,6 +841,7 @@ class ParamsOutTest extends Dsl2Spec {
         out0.isDynamic()
         out0.isPathQualifier()
         out0.channelEmitName == 'aaa'
+        out0.channelTopicName == 'foo'
 
         out1.name == null
         out1.getFilePatterns(ctx,null) == ['hola_99.fa']
@@ -1197,4 +1198,84 @@ class ParamsOutTest extends Dsl2Spec {
         outs[2].inner[1].name == 'bar'
 
     }
+
+
+    def 'should define out with topic' () {
+        setup:
+        def text = '''
+            process hola {
+              output:
+              val x,     topic: ch0
+              env FOO,   topic: ch1
+              path '-',  topic: ch2
+              stdout     topic: ch3    
+              /return/
+            }
+            
+            workflow { hola() }
+            '''
+
+        def binding = [:]
+        def process = parseAndReturnProcess(text, binding)
+
+        when:
+        def outs = process.config.getOutputs() as List
+        then:
+        outs[0].name == 'x'
+        outs[0].channelTopicName == 'ch0'
+        and:
+        outs[1].name == 'FOO'
+        outs[1].channelTopicName == 'ch1'
+        and:
+        outs[2] instanceof StdOutParam  // <-- note: declared as `path`, turned into a `stdout`
+        outs[2].name == '-'
+        outs[2].channelTopicName == 'ch2'
+        and:
+        outs[3] instanceof StdOutParam
+        outs[3].name == '-'
+        outs[3].channelTopicName == 'ch3'
+    }
+
+    def 'should define out tuple with topic'() {
+
+        setup:
+        def text = '''
+            process hola {
+              output:
+                tuple val(x), val(y),   topic: ch1
+                tuple path('foo'),      topic: ch2
+                tuple stdout,env(bar),  topic: ch3
+
+              /return/
+            }
+            
+            workflow { hola() }
+            '''
+
+        def binding = [:]
+        def process = parseAndReturnProcess(text, binding)
+
+        when:
+        def outs = process.config.getOutputs() as List
+
+        then:
+        outs[0].name == 'tupleoutparam<0>'
+        outs[0].channelTopicName == 'ch1'
+        outs[0].inner[0] instanceof ValueOutParam
+        outs[0].inner[0].name == 'x'
+        outs[0].inner[1] instanceof ValueOutParam
+        outs[0].inner[1].name == 'y'
+        and:
+        outs[1].name == 'tupleoutparam<1>'
+        outs[1].channelTopicName == 'ch2'
+        outs[1].inner[0] instanceof FileOutParam
+        and:
+        outs[2].name == 'tupleoutparam<2>'
+        outs[2].channelTopicName == 'ch3'
+        outs[2].inner[0] instanceof StdOutParam
+        outs[2].inner[0].name == '-'
+        outs[2].inner[1] instanceof EnvOutParam
+        outs[2].inner[1].name == 'bar'
+
+    }
 }
diff --git a/modules/nf-commons/build.gradle b/modules/nf-commons/build.gradle
index e3a00932d9..1d09cf4f76 100644
--- a/modules/nf-commons/build.gradle
+++ b/modules/nf-commons/build.gradle
@@ -30,7 +30,7 @@ dependencies {
     api "org.codehaus.groovy:groovy-nio:3.0.19"
     api "commons-lang:commons-lang:2.6"
     api 'com.google.guava:guava:31.1-jre'
-    api 'org.pf4j:pf4j:3.4.1'
+    api 'org.pf4j:pf4j:3.10.0'
     api 'org.pf4j:pf4j-update:2.3.0'
     api 'dev.failsafe:failsafe:3.1.0'
     // patch gson dependency required by pf4j
diff --git a/modules/nf-commons/src/main/nextflow/BuildInfo.groovy b/modules/nf-commons/src/main/nextflow/BuildInfo.groovy
new file mode 100644
index 0000000000..ad236f4ab1
--- /dev/null
+++ b/modules/nf-commons/src/main/nextflow/BuildInfo.groovy
@@ -0,0 +1,88 @@
+/*
+ * Copyright 2013-2023, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package nextflow
+
+import static nextflow.extension.Bolts.*
+
+import java.text.SimpleDateFormat
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+/**
+ * Model app build information
+ * 
+ * @author Paolo Di Tommaso 
+ */
+@Slf4j
+@CompileStatic
+class BuildInfo {
+
+    private static Properties properties
+
+    static {
+        final BUILD_INFO = '/META-INF/build-info.properties'
+        properties = new Properties()
+        try {
+            properties.load( BuildInfo.getResourceAsStream(BUILD_INFO) )
+        }
+        catch( Exception e ) {
+            log.warn "Unable to parse $BUILD_INFO - Cause ${e.message ?: e}"
+        }
+    }
+
+    static Properties getProperties() { properties }
+
+    static String getVersion() { properties.getProperty('version') }
+
+    static String getCommitId() { properties.getProperty('commitId')}
+
+    static String getBuildNum() { properties.getProperty('build') }
+
+    static long getTimestampMillis() { properties.getProperty('timestamp') as long }
+
+    static String getTimestampUTC() {
+        def tz = TimeZone.getTimeZone('UTC')
+        def fmt = new SimpleDateFormat(DATETIME_FORMAT)
+        fmt.setTimeZone(tz)
+        fmt.format(new Date(getTimestampMillis())) + ' ' + tz.getDisplayName( true, TimeZone.SHORT )
+    }
+
+    static String getTimestampLocal() {
+        def tz = TimeZone.getDefault()
+        def fmt = new SimpleDateFormat(DATETIME_FORMAT)
+        fmt.setTimeZone(tz)
+        fmt.format(new Date(getTimestampMillis())) + ' ' + tz.getDisplayName( true, TimeZone.SHORT )
+    }
+
+    static String getTimestampDelta() {
+        if( getTimestampUTC() == getTimestampLocal() ) {
+            return ''
+        }
+
+        final utc = getTimestampUTC().split(' ')
+        final loc = getTimestampLocal().split(' ')
+
+        final result = utc[0] == loc[0] ? loc[1,-1].join(' ') : loc.join(' ')
+        return "($result)"
+    }
+
+    static String getFullVersion() {
+        "${version}_${commitId}"
+    }
+
+}
diff --git a/modules/nf-commons/src/main/nextflow/Const.groovy b/modules/nf-commons/src/main/nextflow/Const.groovy
index 6ad8318823..b7965713e9 100644
--- a/modules/nf-commons/src/main/nextflow/Const.groovy
+++ b/modules/nf-commons/src/main/nextflow/Const.groovy
@@ -15,12 +15,9 @@
  */
 
 package nextflow
+
 import java.nio.file.Path
 import java.nio.file.Paths
-import java.text.SimpleDateFormat
-
-import static nextflow.extension.Bolts.DATETIME_FORMAT
-
 /**
  * Application main constants
  *
@@ -52,54 +49,12 @@ class Const {
     /**
      * The application version
      */
-    static public final String APP_VER = "23.10.0"
+    static public final String APP_VER = "23.11.0-edge"
 
     /**
      * The app build time as linux/unix timestamp
      */
-    static public final long APP_TIMESTAMP = 1697382483238
-
-    /**
-     * The app build number
-     */
-    static public final int APP_BUILDNUM = 5890
-
-    /**
-     * The app build time string relative to UTC timezone
-     */
-    static public final String APP_TIMESTAMP_UTC = {
-
-        def tz = TimeZone.getTimeZone('UTC')
-        def fmt = new SimpleDateFormat(DATETIME_FORMAT)
-        fmt.setTimeZone(tz)
-        fmt.format(new Date(APP_TIMESTAMP)) + ' ' + tz.getDisplayName( true, TimeZone.SHORT )
-
-    } ()
-
-
-    /**
-     * The app build time string relative to local timezone
-     */
-    static public final String APP_TIMESTAMP_LOCAL = {
-
-        def tz = TimeZone.getDefault()
-        def fmt = new SimpleDateFormat(DATETIME_FORMAT)
-        fmt.setTimeZone(tz)
-        fmt.format(new Date(APP_TIMESTAMP)) + ' ' + tz.getDisplayName( true, TimeZone.SHORT )
-
-    } ()
-
-    static String deltaLocal() {
-        def utc = APP_TIMESTAMP_UTC.split(' ')
-        def loc = APP_TIMESTAMP_LOCAL.split(' ')
-
-        if( APP_TIMESTAMP_UTC == APP_TIMESTAMP_LOCAL ) {
-            return ''
-        }
-
-        def result = utc[0] == loc[0] ? loc[1,-1].join(' ') : loc.join(' ')
-        return "($result)"
-    }
+    static public final long APP_TIMESTAMP = 1700857448507
 
 
     private static Path getHomeDir(String appname) {
@@ -113,19 +68,6 @@ class Const {
         return result
     }
 
-    /*
-     * The application 'logo'
-     */
-    static public final String SPLASH =
-
-"""
-      N E X T F L O W
-      version ${APP_VER} build ${APP_BUILDNUM}
-      created ${APP_TIMESTAMP_UTC} ${deltaLocal()}
-      cite doi:10.1038/nbt.3820
-      http://nextflow.io
-"""
-
     static public final String S3_UPLOADER_CLASS = 'nextflow.cloud.aws.nio'
 
     static public final String ROLE_WORKER = 'worker'
diff --git a/modules/nf-commons/src/main/nextflow/plugin/CustomPluginManager.groovy b/modules/nf-commons/src/main/nextflow/plugin/CustomPluginManager.groovy
index 2dcf244478..db6b79e783 100644
--- a/modules/nf-commons/src/main/nextflow/plugin/CustomPluginManager.groovy
+++ b/modules/nf-commons/src/main/nextflow/plugin/CustomPluginManager.groovy
@@ -64,6 +64,6 @@ class CustomPluginManager extends DefaultPluginManager {
 
     @Override
     protected ExtensionFactory createExtensionFactory() {
-        return new SingletonExtensionFactory()
+        return new SingletonExtensionFactory(this)
     }
 }
diff --git a/modules/nf-commons/src/main/nextflow/plugin/PluginUpdater.groovy b/modules/nf-commons/src/main/nextflow/plugin/PluginUpdater.groovy
index 98a5739273..0fab0ba1c5 100644
--- a/modules/nf-commons/src/main/nextflow/plugin/PluginUpdater.groovy
+++ b/modules/nf-commons/src/main/nextflow/plugin/PluginUpdater.groovy
@@ -121,7 +121,7 @@ class PluginUpdater extends UpdateManager {
                 uri = 'file://' + temp.absolutePath
             }
             catch (FileNotFoundException e) {
-                throw new IllegalArgumentException("Provided repository URL does not exists or cannot be accessed: $uri")
+                throw new IllegalArgumentException("Provided repository URL does not exist or cannot be accessed: $uri")
             }
         }
         // create the update repository instance
diff --git a/modules/nf-commons/src/main/nextflow/plugin/PluginsFacade.groovy b/modules/nf-commons/src/main/nextflow/plugin/PluginsFacade.groovy
index b4320d435e..331181d647 100644
--- a/modules/nf-commons/src/main/nextflow/plugin/PluginsFacade.groovy
+++ b/modules/nf-commons/src/main/nextflow/plugin/PluginsFacade.groovy
@@ -397,7 +397,7 @@ class PluginsFacade implements PluginStateListener {
         final bucketDir = config.bucketDir as String
         final executor = Bolts.navigate(config, 'process.executor')
 
-        if( executor == 'awsbatch' || workDir?.startsWith('s3://') || bucketDir?.startsWith('s3://') )
+        if( executor == 'awsbatch' || workDir?.startsWith('s3://') || bucketDir?.startsWith('s3://') || env.containsKey('NXF_ENABLE_AWS_SES') )
             plugins << defaultPlugins.getPlugin('nf-amazon')
 
         if( executor == 'google-lifesciences' || executor == 'google-batch' || workDir?.startsWith('gs://') || bucketDir?.startsWith('gs://')  )
diff --git a/modules/nf-commons/src/main/nextflow/util/StringUtils.groovy b/modules/nf-commons/src/main/nextflow/util/StringUtils.groovy
index 73480b1a5f..8e43acbcaa 100644
--- a/modules/nf-commons/src/main/nextflow/util/StringUtils.groovy
+++ b/modules/nf-commons/src/main/nextflow/util/StringUtils.groovy
@@ -56,7 +56,7 @@ class StringUtils {
         return m.matches() ? m.group(1).toLowerCase() : null
     }
 
-    static private Pattern multilinePattern = ~/"?(password|token|secret|license)"?\s?[:=]\s?"?(\w+)"?/
+    static private Pattern multilinePattern = ~/["']?(password|token|secret|license)["']?\s?[:=]\s?["']?(\w+)["']?/
 
     static String stripSecrets(String message) {
         if (message == null) {
diff --git a/modules/nf-commons/src/test/nextflow/BuildInfoTest.groovy b/modules/nf-commons/src/test/nextflow/BuildInfoTest.groovy
new file mode 100644
index 0000000000..5e4ff2d9bc
--- /dev/null
+++ b/modules/nf-commons/src/test/nextflow/BuildInfoTest.groovy
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2013-2023, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package nextflow
+
+import spock.lang.Specification
+
+/**
+ *
+ * @author Paolo Di Tommaso 
+ */
+class BuildInfoTest extends Specification {
+
+    def 'should load version and commit id' () {
+        expect:
+        BuildInfo.getVersion()
+        BuildInfo.getBuildNum()
+        BuildInfo.getCommitId()
+        BuildInfo.getTimestampMillis()
+    }
+}
diff --git a/modules/nf-commons/src/test/nextflow/plugin/PluginsFacadeTest.groovy b/modules/nf-commons/src/test/nextflow/plugin/PluginsFacadeTest.groovy
index 20eb74ac32..5bb8aa4bf7 100644
--- a/modules/nf-commons/src/test/nextflow/plugin/PluginsFacadeTest.groovy
+++ b/modules/nf-commons/src/test/nextflow/plugin/PluginsFacadeTest.groovy
@@ -4,6 +4,7 @@ import java.nio.file.Files
 import java.nio.file.Paths
 
 import com.sun.net.httpserver.HttpServer
+import nextflow.SysEnv
 import spock.lang.Specification
 import spock.lang.Unroll
 /**
@@ -149,6 +150,8 @@ class PluginsFacadeTest extends Specification {
 
     def 'should return default plugins given config' () {
         given:
+        SysEnv.push([:])
+        and:
         def defaults = new DefaultPlugins(plugins: [
                 'nf-amazon': new PluginSpec('nf-amazon', '0.1.0'),
                 'nf-google': new PluginSpec('nf-google', '0.1.0'),
@@ -187,10 +190,14 @@ class PluginsFacadeTest extends Specification {
         !plugins.find { it.id == 'nf-google' }
         !plugins.find { it.id == 'nf-azure' }
 
+        cleanup:
+        SysEnv.pop()
     }
 
     def 'should return default plugins given workdir' () {
         given:
+        SysEnv.push([:])
+        and:
         def defaults = new DefaultPlugins(plugins: [
                 'nf-amazon': new PluginSpec('nf-amazon', '0.1.0'),
                 'nf-google': new PluginSpec('nf-google', '0.1.0'),
@@ -228,10 +235,14 @@ class PluginsFacadeTest extends Specification {
         !plugins.find { it.id == 'nf-google' }
         !plugins.find { it.id == 'nf-azure' }
 
+        cleanup:
+        SysEnv.pop()
     }
 
     def 'should return default plugins given bucket dir' () {
         given:
+        SysEnv.push([:])
+        and:
         def defaults = new DefaultPlugins(plugins: [
                 'nf-amazon': new PluginSpec('nf-amazon', '0.1.0'),
                 'nf-google': new PluginSpec('nf-google', '0.1.0'),
@@ -269,6 +280,8 @@ class PluginsFacadeTest extends Specification {
         !plugins.find { it.id == 'nf-google' }
         !plugins.find { it.id == 'nf-azure' }
 
+        cleanup:
+        SysEnv.pop()
     }
 
     def 'should get plugins list from env' () {
diff --git a/modules/nf-commons/src/test/nextflow/util/StringUtilsTest.groovy b/modules/nf-commons/src/test/nextflow/util/StringUtilsTest.groovy
index e1e7522902..7feea08f18 100644
--- a/modules/nf-commons/src/test/nextflow/util/StringUtilsTest.groovy
+++ b/modules/nf-commons/src/test/nextflow/util/StringUtilsTest.groovy
@@ -100,13 +100,15 @@ class StringUtilsTest extends Specification {
         StringUtils.stripSecrets(SECRET) == EXPECTED
 
         where:
-        SECRET                                  | EXPECTED
-        'Hi\n here is the "password" : "1234"'  | 'Hi\n here is the "password" : "********"'
-        'Hi\n here is the password : "1"'       | 'Hi\n here is the password : "********"'
-        'Hi\n here is the password : "1"'       | 'Hi\n here is the password : "********"'
-        'Hi\n "password" :"1" \n "token": "123"'| 'Hi\n "password" :"********" \n "token": "********"'
-        'Hi\n password :"1"\nsecret: "345"'     | 'Hi\n password :"********"\nsecret: "********"'
-        'secret="abc" password:"1" more text'   | 'secret="********" password:"********" more text'
+        SECRET                                          | EXPECTED
+        'Hi\n here is the "password" : "1234"'          | 'Hi\n here is the "password" : "********"'
+        'Hi\n here is the password : "1"'               | 'Hi\n here is the password : "********"'
+        'Hi\n here is the password : \'1\''             | 'Hi\n here is the password : \'********\''
+        'Hi\n "password" :"1" \n "token": "123"'        | 'Hi\n "password" :"********" \n "token": "********"'
+        'Hi\n "password" :\'1\' \n "token": "123"'      | 'Hi\n "password" :\'********\' \n "token": "********"'
+        'Hi\n \'password\' :\'1\' \n \'token\': \'123\''| 'Hi\n \'password\' :\'********\' \n \'token\': \'********\''
+        'Hi\n password :"1"\nsecret: "345"'             | 'Hi\n password :"********"\nsecret: "********"'
+        'secret="abc" password:"1" more text'           | 'secret="********" password:"********" more text'
     }
 
     @Unroll
diff --git a/modules/nf-commons/src/testFixtures/groovy/nextflow/plugin/TestPluginDescriptorFinder.groovy b/modules/nf-commons/src/testFixtures/groovy/nextflow/plugin/TestPluginDescriptorFinder.groovy
index 66b40e2669..a7ca0ef139 100644
--- a/modules/nf-commons/src/testFixtures/groovy/nextflow/plugin/TestPluginDescriptorFinder.groovy
+++ b/modules/nf-commons/src/testFixtures/groovy/nextflow/plugin/TestPluginDescriptorFinder.groovy
@@ -19,6 +19,7 @@ package nextflow.plugin
 
 import java.nio.file.Files
 import java.nio.file.Path
+import java.util.jar.Manifest
 
 import org.pf4j.ManifestPluginDescriptorFinder
 /**
@@ -30,12 +31,15 @@ import org.pf4j.ManifestPluginDescriptorFinder
 class TestPluginDescriptorFinder extends ManifestPluginDescriptorFinder {
 
     @Override
-    protected Path getManifestPath(Path pluginPath) {
-        if (Files.isDirectory(pluginPath)) {
-            final manifest = pluginPath.resolve('build/resources/testFixtures/META-INF/MANIFEST.MF')
-            return Files.exists(manifest) ? manifest : null
-        }
+    protected Manifest readManifestFromDirectory(Path pluginPath) {
+        if( !Files.isDirectory(pluginPath) )
+            return null
 
-        return null;
+        final manifestPath = pluginPath.resolve('build/resources/testFixtures/META-INF/MANIFEST.MF')
+        if( !Files.exists(manifestPath) )
+            return null
+
+        final input = Files.newInputStream(manifestPath)
+        return new Manifest(input)
     }
 }
diff --git a/nextflow b/nextflow
index fdbdedccd3..7906c2aa67 100755
--- a/nextflow
+++ b/nextflow
@@ -15,7 +15,7 @@
 #  limitations under the License.
 
 [[ "$NXF_DEBUG" == 'x' ]] && set -x
-NXF_VER=${NXF_VER:-'23.10.0'}
+NXF_VER=${NXF_VER:-'23.11.0-edge'}
 NXF_ORG=${NXF_ORG:-'nextflow-io'}
 NXF_HOME=${NXF_HOME:-$HOME/.nextflow}
 NXF_PROT=${NXF_PROT:-'https'}
diff --git a/nextflow.md5 b/nextflow.md5
index ca1fedf1bf..574adce9f0 100644
--- a/nextflow.md5
+++ b/nextflow.md5
@@ -1 +1 @@
-72d4fc9e2c3d8cf2d1232690b7ea0121
+8a4583bb84d6e8304cc0b1ff5446f21f
diff --git a/nextflow.sha1 b/nextflow.sha1
index 0e653c427c..5214121c92 100644
--- a/nextflow.sha1
+++ b/nextflow.sha1
@@ -1 +1 @@
-9ff8060a99bad698d5ad9316ea3a4acfe2ae51c1
+965e377c383f399cb01e2c109ed2edd8335671e6
diff --git a/nextflow.sha256 b/nextflow.sha256
index 9c595903d8..5fd857dc00 100644
--- a/nextflow.sha256
+++ b/nextflow.sha256
@@ -1 +1 @@
-4b7fba61ecc6d53a6850390bb435455a54ae4d0c3108199f88b16b49e555afdd
+ddb037f134a8289215fa41b45f85e678e4bd45b8da2a3379caf326a99dfbbeeb
diff --git a/packing.gradle b/packing.gradle
index 05993e0465..8587735102 100644
--- a/packing.gradle
+++ b/packing.gradle
@@ -15,7 +15,7 @@ configurations {
 dependencies {
     api project(':nextflow')
     // include Ivy at runtime in order to have Grape @Grab work correctly
-    defaultCfg "org.apache.ivy:ivy:2.5.1"
+    defaultCfg "org.apache.ivy:ivy:2.5.2"
     // default cfg = runtime + httpfs + amazon + tower client + wave client
     defaultCfg project(':nf-httpfs')
     // Capsule manages the fat jar building process
@@ -225,7 +225,7 @@ task deploy( type: Exec, dependsOn: [clean, compile, pack]) {
         checkVersionExits(version)
         
         def path = new File(releaseDir)
-        if( !path.exists() ) error("Releases path does not exists: $path")
+        if( !path.exists() ) error("Releases path does not exist: $path")
         path.eachFile {
             if( it.name.startsWith("nextflow-$version"))
                 files << it
diff --git a/plugins/nf-amazon/build.gradle b/plugins/nf-amazon/build.gradle
index fad4a22db4..51ae9d7b27 100644
--- a/plugins/nf-amazon/build.gradle
+++ b/plugins/nf-amazon/build.gradle
@@ -35,18 +35,18 @@ configurations {
 dependencies {
     compileOnly project(':nextflow')
     compileOnly 'org.slf4j:slf4j-api:2.0.7'
-    compileOnly 'org.pf4j:pf4j:3.4.1'
+    compileOnly 'org.pf4j:pf4j:3.10.0'
 
-    api ('javax.xml.bind:jaxb-api:2.3.1')
-    api ('com.amazonaws:aws-java-sdk-s3:1.12.429')
-    api ('com.amazonaws:aws-java-sdk-ec2:1.12.429')
-    api ('com.amazonaws:aws-java-sdk-batch:1.12.429')
-    api ('com.amazonaws:aws-java-sdk-iam:1.12.429')
-    api ('com.amazonaws:aws-java-sdk-ecs:1.12.429')
-    api ('com.amazonaws:aws-java-sdk-logs:1.12.429')
-    api ('com.amazonaws:aws-java-sdk-codecommit:1.12.429')
-    api ('com.amazonaws:aws-java-sdk-sts:1.12.429')
-    api ('com.amazonaws:aws-java-sdk-ses:1.12.429')
+    api ('javax.xml.bind:jaxb-api:2.4.0-b180830.0359')
+    api ('com.amazonaws:aws-java-sdk-s3:1.12.604')
+    api ('com.amazonaws:aws-java-sdk-ec2:1.12.604')
+    api ('com.amazonaws:aws-java-sdk-batch:1.12.604')
+    api ('com.amazonaws:aws-java-sdk-iam:1.12.604')
+    api ('com.amazonaws:aws-java-sdk-ecs:1.12.604')
+    api ('com.amazonaws:aws-java-sdk-logs:1.12.604')
+    api ('com.amazonaws:aws-java-sdk-codecommit:1.12.604')
+    api ('com.amazonaws:aws-java-sdk-sts:1.12.604')
+    api ('com.amazonaws:aws-java-sdk-ses:1.12.604')
     api ('software.amazon.awssdk:sso:2.20.89')
     api ('software.amazon.awssdk:ssooidc:2.20.89')
 
diff --git a/plugins/nf-amazon/changelog.txt b/plugins/nf-amazon/changelog.txt
index 8696d88720..0ec2acf6c0 100644
--- a/plugins/nf-amazon/changelog.txt
+++ b/plugins/nf-amazon/changelog.txt
@@ -1,5 +1,12 @@
 nf-amazon changelog
 ===================
+2.2.0 - 24 Nov 2023
+- Add support for FUSION_AWS_REGION (#4481) [8f8b09fa]
+- Fix security vulnerabilities (#4513) [a310c777]
+- Fix typos (#4519) [ci fast] [6b1ea726]
+- Fix Fusion symlinks when publishing files (#4348) [89f09fe0]
+- Bump javax.xml.bind:jaxb-api:2.4.0-b180830.0359
+
 2.1.4 - 10 Oct 2023
 - Improve S3 endpoint validation [2b9ae6aa]
 - Add -cloudcache CLI option (#4385) [73fda582]
diff --git a/plugins/nf-amazon/src/main/nextflow/cloud/aws/batch/AwsBatchExecutor.groovy b/plugins/nf-amazon/src/main/nextflow/cloud/aws/batch/AwsBatchExecutor.groovy
index ed04eefefa..70b67f496d 100644
--- a/plugins/nf-amazon/src/main/nextflow/cloud/aws/batch/AwsBatchExecutor.groovy
+++ b/plugins/nf-amazon/src/main/nextflow/cloud/aws/batch/AwsBatchExecutor.groovy
@@ -151,7 +151,7 @@ class AwsBatchExecutor extends Executor implements ExtensionPoint {
         helper = new AwsBatchHelper(client, driver)
         // create the options object
         awsOptions = new AwsOptions(this)
-        log.debug "[AWS BATCH] Executor options=$awsOptions"
+        log.debug "[AWS BATCH] Executor ${awsOptions.fargateMode ? '(FARGATE mode) ' : ''}options=$awsOptions"
     }
 
     /**
diff --git a/plugins/nf-amazon/src/main/nextflow/cloud/aws/batch/AwsBatchTaskHandler.groovy b/plugins/nf-amazon/src/main/nextflow/cloud/aws/batch/AwsBatchTaskHandler.groovy
index b200246ad7..08c175dd9d 100644
--- a/plugins/nf-amazon/src/main/nextflow/cloud/aws/batch/AwsBatchTaskHandler.groovy
+++ b/plugins/nf-amazon/src/main/nextflow/cloud/aws/batch/AwsBatchTaskHandler.groovy
@@ -24,6 +24,7 @@ import java.time.Instant
 
 import com.amazonaws.services.batch.AWSBatch
 import com.amazonaws.services.batch.model.AWSBatchException
+import com.amazonaws.services.batch.model.AssignPublicIp
 import com.amazonaws.services.batch.model.AttemptContainerDetail
 import com.amazonaws.services.batch.model.ClientException
 import com.amazonaws.services.batch.model.ContainerOverrides
@@ -32,6 +33,7 @@ import com.amazonaws.services.batch.model.DescribeJobDefinitionsRequest
 import com.amazonaws.services.batch.model.DescribeJobDefinitionsResult
 import com.amazonaws.services.batch.model.DescribeJobsRequest
 import com.amazonaws.services.batch.model.DescribeJobsResult
+import com.amazonaws.services.batch.model.EphemeralStorage
 import com.amazonaws.services.batch.model.EvaluateOnExit
 import com.amazonaws.services.batch.model.Host
 import com.amazonaws.services.batch.model.JobDefinition
@@ -41,15 +43,18 @@ import com.amazonaws.services.batch.model.JobTimeout
 import com.amazonaws.services.batch.model.KeyValuePair
 import com.amazonaws.services.batch.model.LogConfiguration
 import com.amazonaws.services.batch.model.MountPoint
+import com.amazonaws.services.batch.model.NetworkConfiguration
 import com.amazonaws.services.batch.model.RegisterJobDefinitionRequest
 import com.amazonaws.services.batch.model.RegisterJobDefinitionResult
 import com.amazonaws.services.batch.model.ResourceRequirement
 import com.amazonaws.services.batch.model.ResourceType
 import com.amazonaws.services.batch.model.RetryStrategy
+import com.amazonaws.services.batch.model.RuntimePlatform
 import com.amazonaws.services.batch.model.SubmitJobRequest
 import com.amazonaws.services.batch.model.SubmitJobResult
 import com.amazonaws.services.batch.model.TerminateJobRequest
 import com.amazonaws.services.batch.model.Volume
+import groovy.transform.Canonical
 import groovy.transform.CompileStatic
 import groovy.transform.Memoized
 import groovy.util.logging.Slf4j
@@ -67,6 +72,7 @@ import nextflow.processor.TaskRun
 import nextflow.processor.TaskStatus
 import nextflow.trace.TraceRecord
 import nextflow.util.CacheHelper
+import nextflow.util.MemoryUnit
 /**
  * Implements a task handler for AWS Batch jobs
  */
@@ -204,6 +210,9 @@ class AwsBatchTaskHandler extends TaskHandler implements BatchHandler MAX_ATTEMPTS)
@@ -399,8 +408,10 @@ class AwsBatchTaskHandler extends TaskHandler implements BatchHandler classicSubmitCli() {
         // the cmd list to launch it
         final opts = getAwsOptions()
+        final cmd = opts.s5cmdPath ? s5Cmd(opts) : s3Cmd(opts)
+        return ['bash','-o','pipefail','-c', cmd.toString()]
+    }
+
+    protected String s3Cmd(AwsOptions opts) {
         final cli = opts.getAwsCli()
         final debug = opts.debug ? ' --debug' : ''
         final sse = opts.storageEncryption ? " --sse $opts.storageEncryption" : ''
         final kms = opts.storageKmsKeyId ? " --sse-kms-key-id $opts.storageKmsKeyId" : ''
         final aws = "$cli s3 cp --only-show-errors${sse}${kms}${debug}"
         final cmd = "trap \"{ ret=\$?; $aws ${TaskRun.CMD_LOG} s3:/${getLogFile()}||true; exit \$ret; }\" EXIT; $aws s3:/${getWrapperFile()} - | bash 2>&1 | tee ${TaskRun.CMD_LOG}"
-        return ['bash','-o','pipefail','-c', cmd.toString()]
+        return cmd
+    }
+
+    protected String s5Cmd(AwsOptions opts) {
+        final cli = opts.getS5cmdPath()
+        final sse = opts.storageEncryption ? " --sse $opts.storageEncryption" : ''
+        final kms = opts.storageKmsKeyId ? " --sse-kms-key-id $opts.storageKmsKeyId" : ''
+        final cmd = "trap \"{ ret=\$?; $cli cp${sse}${kms} ${TaskRun.CMD_LOG} s3:/${getLogFile()}||true; exit \$ret; }\" EXIT; $cli cat s3:/${getWrapperFile()} | bash 2>&1 | tee ${TaskRun.CMD_LOG}"
+        return cmd
     }
 
     protected List getSubmitCommand() {
@@ -656,6 +697,7 @@ class AwsBatchTaskHandler extends TaskHandler implements BatchHandler(5)
-        def container = new ContainerOverrides()
+        final container = new ContainerOverrides()
         container.command = getSubmitCommand()
         // set the task memory
-        if( task.config.getMemory() ) {
-            final mega = (int)task.config.getMemory().toMega()
+        final cpus = task.config.getCpus()
+        final mem = task.config.getMemory()
+        if( mem ) {
+            final mega = opts.fargateMode ? normaliseFargateMem(cpus, mem) : mem.toMega()
             if( mega >= 4 )
                 resources << new ResourceRequirement().withType(ResourceType.MEMORY).withValue(mega.toString())
             else
                 log.warn "Ignoring task ${task.lazyName()} memory directive: ${task.config.getMemory()} -- AWS Batch job memory request cannot be lower than 4 MB"
         }
         // set the task cpus
-        if( task.config.getCpus() > 1 )
+        if( cpus > 1 )
             resources << new ResourceRequirement().withType(ResourceType.VCPU).withValue(task.config.getCpus().toString())
 
         final accelerator = task.config.getAccelerator()
@@ -838,5 +882,41 @@ class AwsBatchTaskHandler extends TaskHandler implements BatchHandler FARGATE_MEM = [1 : MemSlot.ofGiga(2,8,1),
+                                                      2 : MemSlot.ofGiga(4, 16, 1),
+                                                      4 : MemSlot.ofGiga(8, 30, 1),
+                                                      8 : MemSlot.ofGiga(16,60, 4),
+                                                      16: MemSlot.ofGiga(32, 120, 8) ]
+
+    protected long normaliseFargateMem(Integer cpus, MemoryUnit mem) {
+        final mega = mem.toMega()
+        final slot = FARGATE_MEM.get(cpus)
+        if( slot==null )
+            new ProcessUnrecoverableException("Requirement of $cpus CPUs is not allowed by Fargate -- Check process with name '${task.lazyName()}'")
+        if( mega <=slot.min ) {
+            log.warn "Process '${task.lazyName()}' memory requirement of ${mem} is below the minimum allowed by Fargate of ${MemoryUnit.of(mega+'MB')}"
+            return slot.min
+        }
+        if( mega >slot.max ) {
+            log.warn "Process '${task.lazyName()}' memory requirement of ${mem} is above the maximum allowed by Fargate of ${MemoryUnit.of(mega+'MB')}"
+            return slot.max
+        }
+        return ceilDiv(mega, slot.step) * slot.step
+    }
+
+    static private long ceilDiv(long x, long y){
+        return -Math.floorDiv(-x,y);
+    }
 }
 
diff --git a/plugins/nf-amazon/src/main/nextflow/cloud/aws/batch/AwsOptions.groovy b/plugins/nf-amazon/src/main/nextflow/cloud/aws/batch/AwsOptions.groovy
index 525da3b8c5..0e9028f433 100644
--- a/plugins/nf-amazon/src/main/nextflow/cloud/aws/batch/AwsOptions.groovy
+++ b/plugins/nf-amazon/src/main/nextflow/cloud/aws/batch/AwsOptions.groovy
@@ -144,4 +144,17 @@ class AwsOptions implements CloudTransferOptions {
         awsConfig.batchConfig.addVolume(path)
         return this
     }
+
+    boolean isFargateMode() {
+        return awsConfig.batchConfig.fargateMode
+    }
+
+    String getS5cmdPath() {
+        return awsConfig.batchConfig.s5cmdPath
+    }
+
+    String getExecutionRole() {
+        return awsConfig.batchConfig.getExecutionRole()
+    }
+
 }
diff --git a/plugins/nf-amazon/src/main/nextflow/cloud/aws/config/AwsBatchConfig.groovy b/plugins/nf-amazon/src/main/nextflow/cloud/aws/config/AwsBatchConfig.groovy
index 6a9552904f..94d5ccad55 100644
--- a/plugins/nf-amazon/src/main/nextflow/cloud/aws/config/AwsBatchConfig.groovy
+++ b/plugins/nf-amazon/src/main/nextflow/cloud/aws/config/AwsBatchConfig.groovy
@@ -79,13 +79,30 @@ class AwsBatchConfig implements CloudTransferOptions {
      */
     private Integer schedulingPriority
 
+    /**
+     * The container execution role
+     */
+    String executionRole
+
+    /**
+     * The path for the `s5cmd` tool as an alternative to `aws s3` CLI to upload/download files
+     */
+    String s5cmdPath
+
+    /**
+     * Whenever it should use Fargate API
+     */
+    boolean fargateMode
+
     /*
      * only for testing
      */
     protected AwsBatchConfig() {}
 
     AwsBatchConfig(Map opts) {
-        cliPath = parseCliPath(opts.cliPath as String)
+        fargateMode = opts.platformType == 'fargate'
+        cliPath = !fargateMode ? parseCliPath(opts.cliPath as String) : null
+        s5cmdPath = fargateMode ? parses5cmdPath(opts.cliPath as String) : null
         maxParallelTransfers = opts.maxParallelTransfers as Integer ?: MAX_TRANSFER
         maxTransferAttempts = opts.maxTransferAttempts as Integer ?: defaultMaxTransferAttempts()
         delayBetweenAttempts = opts.delayBetweenAttempts as Duration ?: DEFAULT_DELAY_BETWEEN_ATTEMPTS
@@ -96,6 +113,7 @@ class AwsBatchConfig implements CloudTransferOptions {
         retryMode = opts.retryMode ?: 'standard'
         shareIdentifier = opts.shareIdentifier
         schedulingPriority = opts.schedulingPriority as Integer ?: 0
+        executionRole = opts.executionRole
         if( retryMode == 'built-in' )
             retryMode = null // this force falling back on NF built-in retry mode instead of delegating to AWS CLI tool
         if( retryMode && retryMode !in AwsOptions.VALID_RETRY_MODES )
@@ -161,6 +179,8 @@ class AwsBatchConfig implements CloudTransferOptions {
     private String parseCliPath(String value) {
         if( !value )
             return null
+        if( value.tokenize('/ ').contains('s5cmd') )
+            return null
         if( !value.startsWith('/') )
             throw new ProcessUnrecoverableException("Not a valid aws-cli tools path: $value -- it must be an absolute path")
         if( !value.endsWith('/bin/aws'))
@@ -195,4 +215,11 @@ class AwsBatchConfig implements CloudTransferOptions {
         return this
     }
 
+    protected String parses5cmdPath(String value) {
+        if( !value )
+            return 's5cmd'
+        if( value.tokenize('/ ').contains('s5cmd') )
+            return value
+        return 's5cmd'
+    }
 }
diff --git a/plugins/nf-amazon/src/main/nextflow/cloud/aws/fusion/AwsFusionEnv.groovy b/plugins/nf-amazon/src/main/nextflow/cloud/aws/fusion/AwsFusionEnv.groovy
index 4c2bcb82d4..30c8584094 100644
--- a/plugins/nf-amazon/src/main/nextflow/cloud/aws/fusion/AwsFusionEnv.groovy
+++ b/plugins/nf-amazon/src/main/nextflow/cloud/aws/fusion/AwsFusionEnv.groovy
@@ -43,6 +43,9 @@ class AwsFusionEnv implements FusionEnv {
         if( creds ) {
             result.AWS_ACCESS_KEY_ID = creds[0]
             result.AWS_SECRET_ACCESS_KEY = creds[1]
+
+            if( creds.size() > 2 )
+                result.AWS_SESSION_TOKEN = creds[2]
         }
         if( endpoint )
             result.AWS_S3_ENDPOINT = endpoint
@@ -59,6 +62,10 @@ class AwsFusionEnv implements FusionEnv {
         final result = awsConfig.getCredentials()
         if( result )
             return result
+
+        if( SysEnv.get('AWS_ACCESS_KEY_ID') && SysEnv.get('AWS_SECRET_ACCESS_KEY') && SysEnv.get('AWS_SESSION_TOKEN') )
+            return List.of(SysEnv.get('AWS_ACCESS_KEY_ID'), SysEnv.get('AWS_SECRET_ACCESS_KEY'), SysEnv.get('AWS_SESSION_TOKEN'))
+
         if( SysEnv.get('AWS_ACCESS_KEY_ID') && SysEnv.get('AWS_SECRET_ACCESS_KEY') )
             return List.of(SysEnv.get('AWS_ACCESS_KEY_ID'), SysEnv.get('AWS_SECRET_ACCESS_KEY'))
         else
diff --git a/plugins/nf-amazon/src/main/nextflow/cloud/aws/nio/S3FileSystemProvider.java b/plugins/nf-amazon/src/main/nextflow/cloud/aws/nio/S3FileSystemProvider.java
index b376ba7532..b0e1e763d9 100644
--- a/plugins/nf-amazon/src/main/nextflow/cloud/aws/nio/S3FileSystemProvider.java
+++ b/plugins/nf-amazon/src/main/nextflow/cloud/aws/nio/S3FileSystemProvider.java
@@ -496,7 +496,7 @@ public void delete(Path path) throws IOException {
 		S3Path s3Path = (S3Path) path;
 
         if (Files.notExists(path)){
-            throw new NoSuchFileException("the path: " + path + " not exists");
+            throw new NoSuchFileException("the path: " + path + " does not exist");
         }
 
         if (Files.isDirectory(path)){
@@ -711,11 +711,11 @@ private S3FileAttributes readAttr0(S3Path s3Path) throws IOException {
 		boolean directory = false;
 		boolean regularFile = false;
 		String key = objectSummary.getKey();
-		// check if is a directory and exists the key of this directory at amazon s3
+		// check if is a directory and the key of this directory exists in amazon s3
 		if (objectSummary.getKey().equals(s3Path.getKey() + "/") && objectSummary.getKey().endsWith("/")) {
 			directory = true;
 		}
-		// is a directory but not exists at amazon s3
+		// is a directory but does not exist in amazon s3
 		else if ((!objectSummary.getKey().equals(s3Path.getKey()) || "".equals(s3Path.getKey())) && objectSummary.getKey().startsWith(s3Path.getKey())){
 			directory = true;
 			// no metadata, we fake one
@@ -910,7 +910,7 @@ private boolean exists(S3Path path) {
 	}
 
 	/**
-	 * Get the Control List, if the path not exists
+	 * Get the Control List, if the path does not exist
      * (because the path is a directory and this key isn't created at amazon s3)
      * then return the ACL of the first child.
      *
diff --git a/plugins/nf-amazon/src/main/nextflow/cloud/aws/nio/util/S3ObjectSummaryLookup.java b/plugins/nf-amazon/src/main/nextflow/cloud/aws/nio/util/S3ObjectSummaryLookup.java
index 9354f8456b..ce1e0e75cd 100644
--- a/plugins/nf-amazon/src/main/nextflow/cloud/aws/nio/util/S3ObjectSummaryLookup.java
+++ b/plugins/nf-amazon/src/main/nextflow/cloud/aws/nio/util/S3ObjectSummaryLookup.java
@@ -37,7 +37,7 @@ public class S3ObjectSummaryLookup {
     private static final Logger log = LoggerFactory.getLogger(S3ObjectSummary.class);
 
     /**
-     * Get the {@link com.amazonaws.services.s3.model.S3ObjectSummary} that represent this Path or her first child if this path not exists
+     * Get the {@link com.amazonaws.services.s3.model.S3ObjectSummary} that represent this Path or its first child if the path does not exist
      * @param s3Path {@link S3Path}
      * @return {@link com.amazonaws.services.s3.model.S3ObjectSummary}
      * @throws java.nio.file.NoSuchFileException if not found the path and any child
@@ -139,7 +139,7 @@ public ObjectMetadata getS3ObjectMetadata(S3Path s3Path) {
     /**
      * get S3Object represented by this S3Path try to access with or without end slash '/'
      * @param s3Path S3Path
-     * @return S3Object or null if not exists
+     * @return S3Object or null if it does not exist
      */
     @Deprecated
     private S3Object getS3Object(S3Path s3Path){
diff --git a/plugins/nf-amazon/src/main/nextflow/cloud/aws/util/S3BashLib.groovy b/plugins/nf-amazon/src/main/nextflow/cloud/aws/util/S3BashLib.groovy
index d203650d50..e203ce0411 100644
--- a/plugins/nf-amazon/src/main/nextflow/cloud/aws/util/S3BashLib.groovy
+++ b/plugins/nf-amazon/src/main/nextflow/cloud/aws/util/S3BashLib.groovy
@@ -17,6 +17,7 @@
 package nextflow.cloud.aws.util
 
 import com.amazonaws.services.s3.model.CannedAccessControlList
+import groovy.transform.CompileStatic
 import nextflow.Global
 import nextflow.Session
 import nextflow.cloud.aws.batch.AwsOptions
@@ -25,6 +26,7 @@ import nextflow.executor.BashFunLib
 /**
  * AWS S3 helper class
  */
+@CompileStatic
 class S3BashLib extends BashFunLib {
 
     private String storageClass = 'STANDARD'
@@ -33,6 +35,7 @@ class S3BashLib extends BashFunLib {
     private String debug = ''
     private String cli = 'aws'
     private String retryMode
+    private String s5cmdPath
     private String acl = ''
 
     S3BashLib withCliPath(String cliPath) {
@@ -70,6 +73,11 @@ class S3BashLib extends BashFunLib {
         return this
     }
 
+    S3BashLib withS5cmdPath(String value) {
+        this.s5cmdPath = value
+        return this
+    }
+    
     S3BashLib withAcl(CannedAccessControlList value) {
         if( value )
             this.acl = "--acl $value "
@@ -86,6 +94,11 @@ class S3BashLib extends BashFunLib {
         """.stripIndent().rightTrim()
     }
 
+    /**
+     * Implement S3 upload/download helper using `aws s3` CLI tool
+     *
+     * @return The Bash script implementing the S3 helper functions
+     */
     protected String s3Lib() {
         """
         # aws helper
@@ -115,8 +128,49 @@ class S3BashLib extends BashFunLib {
         """.stripIndent(true)
     }
 
+    /**
+     * Implement S3 upload/download helper using s3cmd CLI tool
+     * https://github.com/peak/s5cmd
+     *
+     * @return The Bash script implementing the S3 helper functions
+     */
+    protected String s5cmdLib() {
+        final cli = s5cmdPath
+        """
+        # aws helper for s5cmd
+        nxf_s3_upload() {
+            local name=\$1
+            local s3path=\$2
+            if [[ "\$name" == - ]]; then
+              local tmp=\$(nxf_mktemp)
+              cp /dev/stdin \$tmp/\$name
+              $cli cp ${acl}${storageEncryption}${storageKmsKeyId}--storage-class $storageClass \$tmp/\$name "\$s3path"
+            elif [[ -d "\$name" ]]; then
+              $cli cp ${acl}${storageEncryption}${storageKmsKeyId}--storage-class $storageClass "\$name/" "\$s3path/\$name/"
+            else
+              $cli cp ${acl}${storageEncryption}${storageKmsKeyId}--storage-class $storageClass "\$name" "\$s3path/\$name"
+            fi
+        }
+        
+        nxf_s3_download() {
+            local source=\$1
+            local target=\$2
+            local file_name=\$(basename \$1)
+            local is_dir=\$($cli ls \$source | grep -F "DIR \${file_name}/" -c)
+            if [[ \$is_dir == 1 ]]; then
+                $cli cp "\$source/*" "\$target"
+            else 
+                $cli cp "\$source" "\$target"
+            fi
+        }
+        """.stripIndent()
+    }
+
+    @Override
     String render() {
-        super.render() + retryEnv() + s3Lib()
+        return s5cmdPath
+                ? super.render() + s5cmdLib()
+                : super.render() + retryEnv() + s3Lib()
     }
 
     static private S3BashLib lib0(AwsOptions opts, boolean includeCore) {
@@ -131,6 +185,7 @@ class S3BashLib extends BashFunLib {
                 .withStorageKmsKeyId( opts.storageKmsKeyId )
                 .withRetryMode( opts.retryMode )
                 .withDebug( opts.debug )
+                .withS5cmdPath( opts.s5cmdPath )
                 .withAcl( opts.s3Acl )
     }
 
diff --git a/plugins/nf-amazon/src/resources/META-INF/MANIFEST.MF b/plugins/nf-amazon/src/resources/META-INF/MANIFEST.MF
index f206656eb2..41788507b1 100644
--- a/plugins/nf-amazon/src/resources/META-INF/MANIFEST.MF
+++ b/plugins/nf-amazon/src/resources/META-INF/MANIFEST.MF
@@ -1,6 +1,6 @@
 Manifest-Version: 1.0
 Plugin-Class: nextflow.cloud.aws.AmazonPlugin
 Plugin-Id: nf-amazon
-Plugin-Version: 2.1.4
+Plugin-Version: 2.2.0
 Plugin-Provider: Seqera Labs
-Plugin-Requires: >=23.05.0-edge
+Plugin-Requires: >=23.11.0-edge
diff --git a/plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsBatchTaskHandlerTest.groovy b/plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsBatchTaskHandlerTest.groovy
index 2f55259b93..c9568016d4 100644
--- a/plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsBatchTaskHandlerTest.groovy
+++ b/plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsBatchTaskHandlerTest.groovy
@@ -35,14 +35,16 @@ import com.amazonaws.services.batch.model.RetryStrategy
 import com.amazonaws.services.batch.model.SubmitJobRequest
 import com.amazonaws.services.batch.model.SubmitJobResult
 import com.amazonaws.services.batch.model.TerminateJobRequest
-import nextflow.cloud.aws.config.AwsConfig
 import nextflow.Const
+import nextflow.Session
+import nextflow.cloud.aws.config.AwsConfig
 import nextflow.cloud.aws.util.S3PathFactory
 import nextflow.cloud.types.CloudMachineInfo
 import nextflow.cloud.types.PriceModel
 import nextflow.exception.ProcessUnrecoverableException
 import nextflow.executor.Executor
 import nextflow.fusion.FusionScriptLauncher
+import nextflow.processor.Architecture
 import nextflow.processor.BatchContext
 import nextflow.processor.TaskConfig
 import nextflow.processor.TaskProcessor
@@ -50,7 +52,9 @@ import nextflow.processor.TaskRun
 import nextflow.processor.TaskStatus
 import nextflow.script.BaseScript
 import nextflow.script.ProcessConfig
+import nextflow.util.MemoryUnit
 import spock.lang.Specification
+import spock.lang.Unroll
 /**
  *
  * @author Paolo Di Tommaso 
@@ -333,7 +337,7 @@ class AwsBatchTaskHandlerTest extends Specification {
         result = handler.getJobDefinition(task)
         then:
         1 * task.getContainer() >> IMAGE
-        1 * handler.resolveJobDefinition(IMAGE) >> JOB_NAME
+        1 * handler.resolveJobDefinition(task) >> JOB_NAME
         result == JOB_NAME
 
     }
@@ -405,6 +409,7 @@ class AwsBatchTaskHandlerTest extends Specification {
         def JOB_NAME = 'nf-foo-bar-1-0'
         def JOB_ID= '123'
         def handler = Spy(AwsBatchTaskHandler)
+        def task = Mock(TaskRun) { getContainer()>>IMAGE }
 
         def req = Mock(RegisterJobDefinitionRequest) {
             getJobDefinitionName() >> JOB_NAME
@@ -412,17 +417,17 @@ class AwsBatchTaskHandlerTest extends Specification {
         }
 
         when:
-        handler.resolveJobDefinition(IMAGE)
+        handler.resolveJobDefinition(task)
         then:
-        1 * handler.makeJobDefRequest(IMAGE) >> req
+        1 * handler.makeJobDefRequest(task) >> req
         1 * handler.findJobDef(JOB_NAME, JOB_ID) >> null
         1 * handler.createJobDef(req) >> null
 
         when:
-        handler.resolveJobDefinition(IMAGE)
+        handler.resolveJobDefinition(task)
         then:
         // second time are not invoked for the same image
-        1 * handler.makeJobDefRequest(IMAGE) >> req
+        1 * handler.makeJobDefRequest(task) >> req
         0 * handler.findJobDef(JOB_NAME, JOB_ID) >> null
         0 * handler.createJobDef(req) >> null
 
@@ -550,14 +555,15 @@ class AwsBatchTaskHandlerTest extends Specification {
         given:
         def IMAGE = 'foo/bar:1.0'
         def JOB_NAME = 'nf-foo-bar-1-0'
+        def task = Mock(TaskRun) { getContainer()>>IMAGE; getConfig() >> Mock(TaskConfig) }
         def handler = Spy(AwsBatchTaskHandler) {
-            getTask() >> Mock(TaskRun) { getConfig() >> Mock(TaskConfig)  }
+            getTask() >> task
             fusionEnabled() >> false
         }
         handler.@executor = Mock(AwsBatchExecutor)
 
         when:
-        def result = handler.makeJobDefRequest(IMAGE)
+        def result = handler.makeJobDefRequest(task)
         then:
         1 * handler.normalizeJobDefinitionName(IMAGE) >> JOB_NAME
         1 * handler.getAwsOptions() >> new AwsOptions()
@@ -569,7 +575,7 @@ class AwsBatchTaskHandlerTest extends Specification {
         !result.containerProperties.privileged
         
         when:
-        result = handler.makeJobDefRequest(IMAGE)
+        result = handler.makeJobDefRequest(task)
         then:
         1 * handler.normalizeJobDefinitionName(IMAGE) >> JOB_NAME
         1 * handler.getAwsOptions() >> new AwsOptions(awsConfig: new AwsConfig(batch: [cliPath: '/home/conda/bin/aws', logsGroup: '/aws/batch'], region: 'us-east-1'))
@@ -587,18 +593,77 @@ class AwsBatchTaskHandlerTest extends Specification {
 
     }
 
+    def 'should create a fargate job definition' () {
+        given:
+        def ARM64 = new Architecture('linux/arm64')
+        def _100GB = MemoryUnit.of('100GB')
+        def IMAGE = 'foo/bar:1.0'
+        def JOB_NAME = 'nf-foo-bar-1-0'
+        def task = Mock(TaskRun) { getContainer()>>IMAGE }
+        def handler = Spy(AwsBatchTaskHandler) {
+            getTask() >> task
+            fusionEnabled() >> false
+        }
+        handler.@executor = Mock(AwsBatchExecutor)
+        and:
+        def session =  Mock(Session) {
+            getConfig() >> [aws:[batch:[platformType:'fargate', jobRole: 'the-job-role', executionRole: 'the-exec-role']]]
+        }
+        def opts = new AwsOptions(session)
+
+        when:
+        def result = handler.makeJobDefRequest(task)
+        then:
+        task.getConfig() >> Mock(TaskConfig)
+        and:
+        1 * handler.normalizeJobDefinitionName(IMAGE) >> JOB_NAME
+        1 * handler.getAwsOptions() >> opts
+        and:
+        result.jobDefinitionName == JOB_NAME
+        result.type == 'container'
+        result.getPlatformCapabilities() == ['FARGATE']
+        result.containerProperties.getJobRoleArn() == 'the-job-role'
+        result.containerProperties.getExecutionRoleArn() == 'the-exec-role'
+        result.containerProperties.getResourceRequirements().find { it.type=='VCPU'}.getValue() == '1'
+        result.containerProperties.getResourceRequirements().find { it.type=='MEMORY'}.getValue() == '2048'
+        and:
+        result.containerProperties.getEphemeralStorage().sizeInGiB == 50
+        result.containerProperties.getRuntimePlatform() == null
+
+        when:
+        result = handler.makeJobDefRequest(task)
+        then:
+        task.getConfig() >> Mock(TaskConfig) { getDisk()>>_100GB ; getArchitecture()>>ARM64 }
+        and:
+        1 * handler.normalizeJobDefinitionName(IMAGE) >> JOB_NAME
+        1 * handler.getAwsOptions() >> opts
+        and:
+        result.jobDefinitionName == JOB_NAME
+        result.type == 'container'
+        result.getPlatformCapabilities() == ['FARGATE']
+        result.containerProperties.getJobRoleArn() == 'the-job-role'
+        result.containerProperties.getExecutionRoleArn() == 'the-exec-role'
+        result.containerProperties.getResourceRequirements().find { it.type=='VCPU'}.getValue() == '1'
+        result.containerProperties.getResourceRequirements().find { it.type=='MEMORY'}.getValue() == '2048'
+        and:
+        result.containerProperties.getEphemeralStorage().sizeInGiB == 100
+        result.containerProperties.getRuntimePlatform().getCpuArchitecture() == 'ARM64'
+    }
+
     def 'should create a job definition request object for fusion' () {
         given:
         def IMAGE = 'foo/bar:1.0'
         def JOB_NAME = 'nf-foo-bar-1-0'
+        def task = Mock(TaskRun) { getContainer()>>IMAGE; getConfig()>>Mock(TaskConfig)  }
+        and:
         AwsBatchTaskHandler handler = Spy(AwsBatchTaskHandler) {
-            getTask() >> Mock(TaskRun) { getConfig() >> Mock(TaskConfig)  }
+            getTask() >> task
             fusionEnabled() >> true
         }
         handler.@executor = Mock(AwsBatchExecutor) {}
 
         when:
-        def result = handler.makeJobDefRequest(IMAGE)
+        def result = handler.makeJobDefRequest(task)
         then:
         1 * handler.normalizeJobDefinitionName(IMAGE) >> JOB_NAME
         1 * handler.getAwsOptions() >> new AwsOptions()
@@ -615,14 +680,16 @@ class AwsBatchTaskHandlerTest extends Specification {
         def JOB_NAME = 'nf-foo-bar-1-0'
         def executor = Mock(AwsBatchExecutor)
         def opts = Mock(AwsOptions)
+        def task = Mock(TaskRun) { getContainer()>>IMAGE; getConfig()>>Mock(TaskConfig)  }
+        and:
         def handler = Spy(AwsBatchTaskHandler) {
-            getTask() >> Mock(TaskRun) { getConfig() >> Mock(TaskConfig)  }
+            getTask() >> task
             fusionEnabled() >> false
         }
         handler.@executor = executor
 
         when:
-        def result = handler.makeJobDefRequest(IMAGE)
+        def result = handler.makeJobDefRequest(task)
         then:
         1 * handler.normalizeJobDefinitionName(IMAGE) >> JOB_NAME
         1 * handler.getAwsOptions() >> opts
@@ -647,14 +714,16 @@ class AwsBatchTaskHandlerTest extends Specification {
         def JOB_NAME = 'nf-foo-bar-1-0'
         def opts = Mock(AwsOptions)
         def executor = Mock(AwsBatchExecutor)
+        def task = Mock(TaskRun) { getContainer()>>IMAGE; getConfig()>>Mock(TaskConfig) }
+        and:
         def handler = Spy(AwsBatchTaskHandler) {
-            getTask() >> Mock(TaskRun) { getConfig() >> Mock(TaskConfig) }
+            getTask() >> task
             fusionEnabled() >> false
         }
         handler.@executor = executor
 
         when:
-        def result = handler.makeJobDefRequest(IMAGE)
+        def result = handler.makeJobDefRequest(task)
         then:
         1 * handler.normalizeJobDefinitionName(IMAGE) >> JOB_NAME
         1 * handler.getAwsOptions() >> opts
@@ -672,14 +741,16 @@ class AwsBatchTaskHandlerTest extends Specification {
         def opts = Mock(AwsOptions)
         def taskConfig = new TaskConfig(containerOptions: '--privileged --user foo')
         def executor = Mock(AwsBatchExecutor)
+        def task = Mock(TaskRun) { getContainer()>>IMAGE; getConfig()>>taskConfig }
+        and:
         def handler = Spy(AwsBatchTaskHandler) {
-            getTask() >> Mock(TaskRun) { getConfig() >> taskConfig }
+            getTask() >> task
             fusionEnabled() >> false
         }
         handler.@executor = executor
 
         when:
-        def result = handler.makeJobDefRequest(IMAGE)
+        def result = handler.makeJobDefRequest(task)
         then:
         1 * handler.normalizeJobDefinitionName(IMAGE) >> JOB_NAME
         1 * handler.getAwsOptions() >> opts
@@ -872,6 +943,36 @@ class AwsBatchTaskHandlerTest extends Specification {
 
     }
 
+    def 'should render submit command with s5cmd' () {
+        given:
+        def handler = Spy(AwsBatchTaskHandler) {
+            fusionEnabled() >> false
+        }
+
+        when:
+        def result =  handler.getSubmitCommand()
+        then:
+        handler.getAwsOptions() >> Mock(AwsOptions)  { getS5cmdPath() >> 's5cmd' }
+        handler.getLogFile() >> Paths.get('/work/log')
+        handler.getWrapperFile() >> Paths.get('/work/run')
+        then:
+        result.join(' ') == 'bash -o pipefail -c trap "{ ret=$?; s5cmd cp .command.log s3://work/log||true; exit $ret; }" EXIT; s5cmd cat s3://work/run | bash 2>&1 | tee .command.log'
+
+        when:
+        result =  handler.getSubmitCommand()
+        then:
+        handler.getAwsOptions() >> Mock(AwsOptions)  {
+            getS5cmdPath() >> 's5cmd --debug'
+            getStorageEncryption() >> 'aws:kms'
+            getStorageKmsKeyId() >> 'kms-key-123'
+        }
+        handler.getLogFile() >> Paths.get('/work/log')
+        handler.getWrapperFile() >> Paths.get('/work/run')
+        then:
+        result.join(' ') == 'bash -o pipefail -c trap "{ ret=$?; s5cmd --debug cp --sse aws:kms --sse-kms-key-id kms-key-123 .command.log s3://work/log||true; exit $ret; }" EXIT; s5cmd --debug cat s3://work/run | bash 2>&1 | tee .command.log'
+
+    }
+
     def 'should create an aws submit request with labels'() {
 
         given:
@@ -922,4 +1023,47 @@ class AwsBatchTaskHandlerTest extends Specification {
         result.join(' ') == '/usr/bin/fusion bash /fusion/s3/my-bucket/work/dir/.command.run'
     }
 
+    @Unroll
+    def 'should normalise fargate mem' () {
+        given:
+        def handler = Spy(AwsBatchTaskHandler) {
+            getTask() >> Mock(TaskRun) { lazyName() >> 'foo' }
+        }
+        expect:
+        handler.normaliseFargateMem(CPUS, MemoryUnit.of( MEM * 1024L*1024L )) == EXPECTED
+
+        where:
+        CPUS    | MEM       | EXPECTED
+        1       | 100       | 2048
+        1       | 1000      | 2048
+        1       | 2000      | 2048
+        1       | 3000      | 3072
+        1       | 7000      | 7168
+        1       | 8000      | 8192
+        1       | 10000     | 8192
+        and:
+        2       | 1000      | 4096
+        2       | 6000      | 6144
+        2       | 16000     | 16384
+        2       | 20000     | 16384
+        and:
+        4       | 1000      | 8192
+        4       | 8000      | 8192
+        4       | 16000     | 16384
+        4       | 30000     | 30720
+        4       | 40000     | 30720
+        and:
+        8       | 1000      | 16384
+        8       | 10000     | 16384
+        8       | 20000     | 20480
+        8       | 30000     | 32768
+        8       | 100000    | 61440
+        and:
+        16      | 1000      | 32768
+        16      | 30000     | 32768
+        16      | 40000     | 40960
+        16      | 60000     | 65536
+        16      | 100000    | 106496
+        16      | 200000    | 122880
+    }
 }
diff --git a/plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsOptionsTest.groovy b/plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsOptionsTest.groovy
index de843c1700..8d7a743ebe 100644
--- a/plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsOptionsTest.groovy
+++ b/plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsOptionsTest.groovy
@@ -222,6 +222,27 @@ class AwsOptionsTest extends Specification {
         opts.volumes == ['/some/dir', '/other/dir']
     }
 
+    @Unroll
+    def 'should get aws cli path' () {
+        def session = new Session(CONFIG)
+
+        when:
+        def opts = new AwsOptions(session)
+        then:
+        opts.cliPath == S3CLI_PATH
+        opts.s5cmdPath == S5CMD_PATH
+
+        where:
+        CONFIG                                                                      | S3CLI_PATH        | S5CMD_PATH
+        [aws:[batch:[:]]]                                                           | null              | null
+        [aws:[batch:[cliPath: '/usr/bin/aws']]]                                     | '/usr/bin/aws'    | null
+        [aws:[batch:[cliPath: 's5cmd']]]                                            | null              | null
+        [aws:[batch:[platformType: 'fargate', cliPath: 's5cmd']]]                   | null              | 's5cmd'
+        [aws:[batch:[platformType: 'fargate', cliPath: '/some/path/s5cmd']]]        | null              | '/some/path/s5cmd'
+        [aws:[batch:[platformType: 'fargate', cliPath: 's5cmd --foo']]]             | null              | 's5cmd --foo'
+        [aws:[batch:[platformType: 'fargate', cliPath: '/some/path/s5cmd --foo']]]  | null              | '/some/path/s5cmd --foo'
+    }
+    
     def 'should parse s3 acl' ( ) {
         when:
         def opts = new AwsOptions(new Session(aws:[client:[s3Acl: 'PublicRead']]))
diff --git a/plugins/nf-amazon/src/test/nextflow/cloud/aws/config/AwsBatchConfigTest.groovy b/plugins/nf-amazon/src/test/nextflow/cloud/aws/config/AwsBatchConfigTest.groovy
index a86fc5b3b5..18a1c6b71a 100644
--- a/plugins/nf-amazon/src/test/nextflow/cloud/aws/config/AwsBatchConfigTest.groovy
+++ b/plugins/nf-amazon/src/test/nextflow/cloud/aws/config/AwsBatchConfigTest.groovy
@@ -40,8 +40,11 @@ class AwsBatchConfigTest extends Specification {
         !batch.cliPath
         !batch.volumes
         !batch.jobRole
+        !batch.executionRole
         !batch.logsGroup
         !batch.shareIdentifier
+        !batch.isFargateMode()
+        !batch.s5cmdPath
         batch.schedulingPriority == 0
     }
 
@@ -55,6 +58,7 @@ class AwsBatchConfigTest extends Specification {
                 maxSpotAttempts: 4,
                 volumes: '/some/path:/mnt/path,/other/path',
                 jobRole: 'xyz',
+                executionRole: 'some:exec:role',
                 logsGroup: 'group-name-123',
                 retryMode: 'legacy',
                 shareIdentifier: 'id-x1',
@@ -71,10 +75,12 @@ class AwsBatchConfigTest extends Specification {
         batch.maxSpotAttempts == 4
         batch.volumes == ['/some/path:/mnt/path', '/other/path']
         batch.jobRole == 'xyz'
+        batch.executionRole == 'some:exec:role'
         batch.logsGroup == 'group-name-123'
         batch.retryMode == 'legacy'
         batch.shareIdentifier == 'id-x1'
         batch.schedulingPriority == 100
+        !batch.isFargateMode()
     }
 
     def 'should parse volumes list' () {
@@ -112,4 +118,25 @@ class AwsBatchConfigTest extends Specification {
         opts.volumes == ['/some/dir', '/other/dir']
     }
 
+    def 'should parse cli path' () {
+        given:
+        def opts = new AwsBatchConfig(OPTS)
+
+        expect:
+        opts.cliPath == S3_CLI_PATH
+        opts.s5cmdPath == S5_CLI_PATH
+        opts.isFargateMode() == FARGATE
+        
+        where:
+        OPTS                                                        | S3_CLI_PATH       | S5_CLI_PATH       | FARGATE
+        [:]                                                         | null              | null              | false
+        [cliPath: "/opt/bin/aws"]                                   | '/opt/bin/aws'    | null              | false
+        [cliPath: "/s5cmd"]                                         | null              | null              | false
+        [cliPath: "/opt/s5cmd --foo"]                               | null              | null              | false
+        and:
+        [platformType: 'fargate', cliPath: "/opt/bin/aws"]          | null              | 's5cmd'           | true
+        [platformType: 'fargate', cliPath: "/opt/s5cmd"]            | null              | '/opt/s5cmd'      | true
+        [platformType: 'fargate', cliPath: "/opt/s5cmd --foo"]      | null              | '/opt/s5cmd --foo'| true
+    }
+
 }
diff --git a/plugins/nf-amazon/src/test/nextflow/cloud/aws/fusion/AwsFusionEnvTest.groovy b/plugins/nf-amazon/src/test/nextflow/cloud/aws/fusion/AwsFusionEnvTest.groovy
index dfa15d661e..feba870298 100644
--- a/plugins/nf-amazon/src/test/nextflow/cloud/aws/fusion/AwsFusionEnvTest.groovy
+++ b/plugins/nf-amazon/src/test/nextflow/cloud/aws/fusion/AwsFusionEnvTest.groovy
@@ -77,4 +77,28 @@ class AwsFusionEnvTest extends Specification {
         cleanup:
         Global.config = null
     }
+
+    def 'should return env environment with session token' () {
+        given:
+        SysEnv.push([AWS_ACCESS_KEY_ID: 'x1', AWS_SECRET_ACCESS_KEY: 'y1', AWS_S3_ENDPOINT: 'http://my-host.com', AWS_SESSION_TOKEN: 'z1'])
+        and:
+
+        when:
+        def config = Mock(FusionConfig)
+        def env = new AwsFusionEnv().getEnvironment('s3', Mock(FusionConfig))
+        then:
+        env == [AWS_S3_ENDPOINT:'http://my-host.com']
+
+        when:
+        config = Mock(FusionConfig) { exportStorageCredentials() >> true }
+        env = new AwsFusionEnv().getEnvironment('s3', config)
+        then:
+        env == [AWS_ACCESS_KEY_ID: 'x1',
+                AWS_SECRET_ACCESS_KEY: 'y1',
+                AWS_S3_ENDPOINT:'http://my-host.com',
+                AWS_SESSION_TOKEN: 'z1']
+
+        cleanup:
+        SysEnv.pop()
+    }
 }
diff --git a/plugins/nf-amazon/src/test/nextflow/cloud/aws/util/S3BashLibTest.groovy b/plugins/nf-amazon/src/test/nextflow/cloud/aws/util/S3BashLibTest.groovy
index f3c3608a18..524dda71ee 100644
--- a/plugins/nf-amazon/src/test/nextflow/cloud/aws/util/S3BashLibTest.groovy
+++ b/plugins/nf-amazon/src/test/nextflow/cloud/aws/util/S3BashLibTest.groovy
@@ -742,4 +742,80 @@ class S3BashLibTest extends Specification {
             }
             '''.stripIndent(true)
     }
+
+    def 'should create s5cmd script' () {
+        given:
+        Global.session = Mock(Session) {
+            getConfig() >> [aws:[batch:[platformType: 'fargate', cliPath: 's5cmd']]]
+        }
+
+        expect:
+        S3BashLib.script()  == '''
+            # aws helper for s5cmd
+            nxf_s3_upload() {
+                local name=$1
+                local s3path=$2
+                if [[ "$name" == - ]]; then
+                  local tmp=$(nxf_mktemp)
+                  cp /dev/stdin $tmp/$name
+                  s5cmd cp --storage-class STANDARD $tmp/$name "$s3path"
+                elif [[ -d "$name" ]]; then
+                  s5cmd cp --storage-class STANDARD "$name/" "$s3path/$name/"
+                else
+                  s5cmd cp --storage-class STANDARD "$name" "$s3path/$name"
+                fi
+            }
+            
+            nxf_s3_download() {
+                local source=$1
+                local target=$2
+                local file_name=$(basename $1)
+                local is_dir=$(s5cmd ls $source | grep -F "DIR ${file_name}/" -c)
+                if [[ $is_dir == 1 ]]; then
+                    s5cmd cp "$source/*" "$target"
+                else 
+                    s5cmd cp "$source" "$target"
+                fi
+            }
+            '''.stripIndent(true)
+    }
+
+    def 'should create s5cmd script with acl' () {
+        given:
+        Global.session = Mock(Session) {
+            getConfig() >> [aws:[batch:[platformType: 'fargate', cliPath: 's5cmd'], client:[ s3Acl: 'PublicRead']]]
+        }
+
+        expect:
+        S3BashLib.script()  == '''
+            # aws helper for s5cmd
+            nxf_s3_upload() {
+                local name=$1
+                local s3path=$2
+                if [[ "$name" == - ]]; then
+                  local tmp=$(nxf_mktemp)
+                  cp /dev/stdin $tmp/$name
+                  s5cmd cp --acl public-read --storage-class STANDARD $tmp/$name "$s3path"
+                elif [[ -d "$name" ]]; then
+                  s5cmd cp --acl public-read --storage-class STANDARD "$name/" "$s3path/$name/"
+                else
+                  s5cmd cp --acl public-read --storage-class STANDARD "$name" "$s3path/$name"
+                fi
+            }
+            
+            nxf_s3_download() {
+                local source=$1
+                local target=$2
+                local file_name=$(basename $1)
+                local is_dir=$(s5cmd ls $source | grep -F "DIR ${file_name}/" -c)
+                if [[ $is_dir == 1 ]]; then
+                    s5cmd cp "$source/*" "$target"
+                else 
+                    s5cmd cp "$source" "$target"
+                fi
+            }
+            '''.stripIndent(true)
+    }
+    
+
 }
diff --git a/plugins/nf-amazon/src/test/nextflow/processor/PublishDirS3Test.groovy b/plugins/nf-amazon/src/test/nextflow/processor/PublishDirS3Test.groovy
index 460de36581..4e92cb97e7 100644
--- a/plugins/nf-amazon/src/test/nextflow/processor/PublishDirS3Test.groovy
+++ b/plugins/nf-amazon/src/test/nextflow/processor/PublishDirS3Test.groovy
@@ -16,14 +16,14 @@
 
 package nextflow.processor
 
+import java.nio.file.FileSystems
 import java.nio.file.Files
 
+import nextflow.Global
+import nextflow.Session
 import nextflow.cloud.aws.nio.S3Path
-import spock.lang.Specification
-
-import java.nio.file.FileSystems
-
 import nextflow.file.FileHelper
+import spock.lang.Specification
 
 /**
  *
@@ -73,4 +73,30 @@ class PublishDirS3Test extends Specification {
         folder?.deleteDir()
     }
 
+    def 'should resolve fusion symlinks' () {
+        given:
+        Global.session = Mock(Session) {
+            config >> [fusion: [enabled: true]]
+        }
+        and:
+        def prev = FileHelper.asPath('s3://bucket/work/0/foo.txt')
+        def file = FileHelper.asPath('s3://bucket/work/1/foo.txt')
+        def taskInputs = ['foo.txt': file]
+        and:
+        def targetDir = FileHelper.asPath('s3://bucket/results')
+        def target = targetDir.resolve('foo.txt')
+        def publisher = Spy(new PublishDir(path: targetDir)) { getTaskInputs()>>taskInputs }
+
+        when:
+        publisher.processFile(file, target)
+        then:
+        1 * publisher.resolveFusionLink(file) >> prev
+        _ * publisher.makeDirs(target.parent) >> _
+        1 * publisher.processFileImpl(prev, target) >> _
+        1 * publisher.notifyFilePublish(target, prev) >> _
+
+        cleanup:
+        Global.session = null
+    }
+
 }
diff --git a/plugins/nf-azure/build.gradle b/plugins/nf-azure/build.gradle
index c74ef70fe8..284e994fa4 100644
--- a/plugins/nf-azure/build.gradle
+++ b/plugins/nf-azure/build.gradle
@@ -35,7 +35,7 @@ configurations {
 dependencies {
     compileOnly project(':nextflow')
     compileOnly 'org.slf4j:slf4j-api:2.0.7'
-    compileOnly 'org.pf4j:pf4j:3.4.1'
+    compileOnly 'org.pf4j:pf4j:3.10.0'
     api('com.azure:azure-storage-blob:12.22.1') {
         exclude group: 'org.slf4j', module: 'slf4j-api'
     }
diff --git a/plugins/nf-azure/changelog.txt b/plugins/nf-azure/changelog.txt
index cc42b1938b..409ba65056 100644
--- a/plugins/nf-azure/changelog.txt
+++ b/plugins/nf-azure/changelog.txt
@@ -1,5 +1,9 @@
 nf-azure changelog
 ===================
+1.4.0 - 24 Nov 2023
+- Fix security vulnerabilities (#4513) [a310c777]
+- Add support for Azure low-priority pool (#4527) [8320ea10]
+
 1.3.2 - 28 Sep 2023
 - Retry TimeoutException in azure file system (#4295) [79248355]
 
diff --git a/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzBatchService.groovy b/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzBatchService.groovy
index a16506e4ad..170a0f6e11 100644
--- a/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzBatchService.groovy
+++ b/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzBatchService.groovy
@@ -749,7 +749,12 @@ class AzBatchService implements Closeable {
                     .withAutoScaleEvaluationInterval( new Period().withSeconds(interval) )
                     .withAutoScaleFormula(scaleFormula(spec.opts))
         }
-        else {
+        else if( spec.opts.lowPriority ) {
+            log.debug "Creating low-priority pool with id: ${spec.poolId}; vmCount=${spec.opts.vmCount};"
+            poolParams
+                .withTargetLowPriorityNodes(spec.opts.vmCount)
+        }
+        else  {
             log.debug "Creating fixed pool with id: ${spec.poolId}; vmCount=${spec.opts.vmCount};"
             poolParams
                     .withTargetDedicatedNodes(spec.opts.vmCount)
@@ -759,23 +764,24 @@ class AzBatchService implements Closeable {
     }
 
     protected String scaleFormula(AzPoolOpts opts) {
+        final target = opts.lowPriority ? 'TargetLowPriorityNodes' : 'TargetDedicatedNodes'
         // https://docs.microsoft.com/en-us/azure/batch/batch-automatic-scaling
-        def DEFAULT_FORMULA = '''
+        final DEFAULT_FORMULA = """
             // Get pool lifetime since creation.
             lifespan = time() - time("{{poolCreationTime}}");
             interval = TimeInterval_Minute * {{scaleInterval}};
             
             // Compute the target nodes based on pending tasks.
-            // $PendingTasks == The sum of $ActiveTasks and $RunningTasks
-            $samples = $PendingTasks.GetSamplePercent(interval);
-            $tasks = $samples < 70 ? max(0, $PendingTasks.GetSample(1)) : max( $PendingTasks.GetSample(1), avg($PendingTasks.GetSample(interval)));
-            $targetVMs = $tasks > 0 ? $tasks : max(0, $TargetDedicatedNodes/2);
-            targetPoolSize = max(0, min($targetVMs, {{maxVmCount}}));
+            // \$PendingTasks == The sum of \$ActiveTasks and \$RunningTasks
+            \$samples = \$PendingTasks.GetSamplePercent(interval);
+            \$tasks = \$samples < 70 ? max(0, \$PendingTasks.GetSample(1)) : max( \$PendingTasks.GetSample(1), avg(\$PendingTasks.GetSample(interval)));
+            \$targetVMs = \$tasks > 0 ? \$tasks : max(0, \$TargetDedicatedNodes/2);
+            targetPoolSize = max(0, min(\$targetVMs, {{maxVmCount}}));
             
             // For first interval deploy 1 node, for other intervals scale up/down as per tasks.
-            $TargetDedicatedNodes = lifespan < interval ? {{vmCount}} : targetPoolSize;
-            $NodeDeallocationOption = taskcompletion;
-            '''.stripIndent(true)
+            \$${target} = lifespan < interval ? {{vmCount}} : targetPoolSize;
+            \$NodeDeallocationOption = taskcompletion;
+            """.stripIndent(true)
 
         final scaleFormula = opts.scaleFormula ?: DEFAULT_FORMULA
         final vars = poolCreationBindings(opts, Instant.now())
diff --git a/plugins/nf-azure/src/main/nextflow/cloud/azure/config/AzPoolOpts.groovy b/plugins/nf-azure/src/main/nextflow/cloud/azure/config/AzPoolOpts.groovy
index 178c34e47f..c17b818c9d 100644
--- a/plugins/nf-azure/src/main/nextflow/cloud/azure/config/AzPoolOpts.groovy
+++ b/plugins/nf-azure/src/main/nextflow/cloud/azure/config/AzPoolOpts.groovy
@@ -66,6 +66,7 @@ class AzPoolOpts implements CacheFunnel {
     String password
 
     String virtualNetwork
+    boolean lowPriority
 
     AzPoolOpts() {
         this(Collections.emptyMap())
@@ -89,6 +90,7 @@ class AzPoolOpts implements CacheFunnel {
         this.userName = opts.userName
         this.password = opts.password
         this.virtualNetwork = opts.virtualNetwork
+        this.lowPriority = opts.lowPriority as boolean
     }
 
     @Override
@@ -108,6 +110,7 @@ class AzPoolOpts implements CacheFunnel {
         hasher.putUnencodedChars(scaleFormula ?: '')
         hasher.putUnencodedChars(schedulePolicy ?: '')
         hasher.putUnencodedChars(virtualNetwork ?: '')
+        hasher.putBoolean(lowPriority)
         return hasher
     }
 
diff --git a/plugins/nf-azure/src/resources/META-INF/MANIFEST.MF b/plugins/nf-azure/src/resources/META-INF/MANIFEST.MF
index d9273f2629..46261512b7 100644
--- a/plugins/nf-azure/src/resources/META-INF/MANIFEST.MF
+++ b/plugins/nf-azure/src/resources/META-INF/MANIFEST.MF
@@ -1,6 +1,6 @@
 Manifest-Version: 1.0
 Plugin-Class: nextflow.cloud.azure.AzurePlugin
 Plugin-Id: nf-azure
-Plugin-Version: 1.3.2
+Plugin-Version: 1.4.0
 Plugin-Provider: Seqera Labs
-Plugin-Requires: >=23.01.0-edge
+Plugin-Requires: >=23.11.0-edge
diff --git a/plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzBatchServiceTest.groovy b/plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzBatchServiceTest.groovy
index 0b00cc9a38..10c2a6836f 100644
--- a/plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzBatchServiceTest.groovy
+++ b/plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzBatchServiceTest.groovy
@@ -237,6 +237,18 @@ class AzBatchServiceTest extends Specification {
         formula.contains '$TargetDedicatedNodes = lifespan < interval ? 3 : targetPoolSize;'
     }
 
+    def 'should check scaling formula for low-priority' () {
+        given:
+        def exec = Mock(AzBatchExecutor) { getConfig() >> new AzConfig([:]) }
+        def svc = new AzBatchService(exec)
+
+        when:
+        def formula = svc.scaleFormula( new AzPoolOpts(lowPriority: true, vmCount: 3, maxVmCount: 10, scaleInterval: Duration.of('5 min')) )
+        then:
+        formula.contains 'interval = TimeInterval_Minute * 5;'
+        formula.contains '$TargetLowPriorityNodes = lifespan < interval ? 3 : targetPoolSize;'
+    }
+
     def 'should  check formula vars' () {
         given:
         def exec = Mock(AzBatchExecutor) { getConfig() >> new AzConfig([:]) }
@@ -347,7 +359,7 @@ class AzBatchServiceTest extends Specification {
         then:
         1 * svc.guessBestVm(LOC, CPUS, MEM, TYPE) >> VM
         and:
-        spec.poolId == 'nf-pool-9022a3fbfb5f93028d78fefaea5e21ab-Standard_X1'
+        spec.poolId == 'nf-pool-6e9cf97d3d846621464131d3842265ce-Standard_X1'
         spec.metadata == [foo: 'bar']
 
     }
diff --git a/plugins/nf-azure/src/test/nextflow/cloud/azure/config/AzPoolOptsTest.groovy b/plugins/nf-azure/src/test/nextflow/cloud/azure/config/AzPoolOptsTest.groovy
new file mode 100644
index 0000000000..ca57b5f4ec
--- /dev/null
+++ b/plugins/nf-azure/src/test/nextflow/cloud/azure/config/AzPoolOptsTest.groovy
@@ -0,0 +1,94 @@
+/*
+ * Copyright 2013-2023, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package nextflow.cloud.azure.config
+
+import nextflow.util.Duration
+import spock.lang.Specification
+/**
+ *
+ * @author Paolo Di Tommaso 
+ */
+class AzPoolOptsTest extends Specification {
+
+    def 'should create pool options' () {
+        when:
+        def opts = new AzPoolOpts()
+        then:
+        !opts.runAs
+        !opts.privileged
+        opts.publisher == AzPoolOpts.DEFAULT_PUBLISHER
+        opts.offer == AzPoolOpts.DEFAULT_OFFER
+        opts.sku == AzPoolOpts.DEFAULT_SKU
+        opts.vmType == AzPoolOpts.DEFAULT_VM_TYPE
+        opts.fileShareRootPath == '/mnt/batch/tasks/fsmounts'
+        opts.vmCount == 1
+        !opts.autoScale
+        !opts.scaleFormula
+        !opts.schedulePolicy
+        opts.scaleInterval == AzPoolOpts.DEFAULT_SCALE_INTERVAL
+        opts.maxVmCount == opts.vmCount *3
+        !opts.registry
+        !opts.userName
+        !opts.password
+        !opts.virtualNetwork
+        !opts.lowPriority
+    }
+
+    def 'should create pool with custom options' () {
+        when:
+        def opts = new AzPoolOpts([
+            runAs:'foo',
+            privileged: true,
+            publisher: 'some-pub',
+            offer: 'some-offer',
+            sku: 'some-sku',
+            vmType: 'some-vmtype',
+            vmCount: 10,
+            autoScale: true,
+            scaleFormula: 'some-formula',
+            schedulePolicy: 'some-policy',
+            scaleInterval: Duration.of('10s'),
+            maxVmCount: 100,
+            registry: 'some-reg',
+            userName: 'some-user',
+            password: 'some-pwd',
+            virtualNetwork: 'some-vnet',
+            lowPriority: true
+        ])
+        then:
+        opts.runAs == 'foo'
+        opts.privileged
+        opts.publisher == 'some-pub'
+        opts.offer == 'some-offer'
+        opts.sku == 'some-sku'
+        opts.vmType == 'some-vmtype'
+        opts.fileShareRootPath == ''
+        opts.vmCount == 10
+        opts.autoScale
+        opts.scaleFormula == 'some-formula'
+        opts.schedulePolicy == 'some-policy'
+        opts.scaleInterval == Duration.of('10s')
+        opts.maxVmCount == 100
+        opts.registry == 'some-reg'
+        opts.userName == 'some-user'
+        opts.password == 'some-pwd'
+        opts.virtualNetwork == 'some-vnet'
+        opts.lowPriority
+    }
+
+}
diff --git a/plugins/nf-cloudcache/build.gradle b/plugins/nf-cloudcache/build.gradle
index d37b5354c9..d8ebde2e0a 100644
--- a/plugins/nf-cloudcache/build.gradle
+++ b/plugins/nf-cloudcache/build.gradle
@@ -32,7 +32,7 @@ configurations {
 dependencies {
     compileOnly project(':nextflow')
     compileOnly 'org.slf4j:slf4j-api:2.0.7'
-    compileOnly 'org.pf4j:pf4j:3.4.1'
+    compileOnly 'org.pf4j:pf4j:3.10.0'
 
     testImplementation(testFixtures(project(":nextflow")))
     testImplementation "org.codehaus.groovy:groovy:3.0.19"
diff --git a/plugins/nf-cloudcache/changelog.txt b/plugins/nf-cloudcache/changelog.txt
index 05fd5fad8e..20dbd6679a 100644
--- a/plugins/nf-cloudcache/changelog.txt
+++ b/plugins/nf-cloudcache/changelog.txt
@@ -1,5 +1,8 @@
 nf-cloudcache changelog
 =======================
+0.3.1 - 24 Nov 2023
+- Fix security vulnerabilities (#4513) [a310c777]
+
 0.3.0 - 10 Oct 2023
 - Add -cloudcache CLI option (#4385) [73fda582]
 
diff --git a/plugins/nf-cloudcache/src/resources/META-INF/MANIFEST.MF b/plugins/nf-cloudcache/src/resources/META-INF/MANIFEST.MF
index b4267c3b61..2d9e60d1d7 100644
--- a/plugins/nf-cloudcache/src/resources/META-INF/MANIFEST.MF
+++ b/plugins/nf-cloudcache/src/resources/META-INF/MANIFEST.MF
@@ -1,6 +1,6 @@
 Manifest-Version: 1.0
 Plugin-Class: nextflow.CloudCachePlugin
 Plugin-Id: nf-cloudcache
-Plugin-Version: 0.3.0
+Plugin-Version: 0.3.1
 Plugin-Provider: Seqera Labs
-Plugin-Requires: >=23.04.0
+Plugin-Requires: >=23.11.0-edge
diff --git a/plugins/nf-codecommit/build.gradle b/plugins/nf-codecommit/build.gradle
index 0137c25c5a..b7402d77a3 100644
--- a/plugins/nf-codecommit/build.gradle
+++ b/plugins/nf-codecommit/build.gradle
@@ -35,9 +35,9 @@ configurations {
 dependencies {
     compileOnly project(':nextflow')
     compileOnly 'org.slf4j:slf4j-api:2.0.7'
-    compileOnly 'org.pf4j:pf4j:3.4.1'
+    compileOnly 'org.pf4j:pf4j:3.10.0'
 
-    api ('javax.xml.bind:jaxb-api:2.3.1')
+    api ('javax.xml.bind:jaxb-api:2.4.0-b180830.0359')
     api ('com.amazonaws:aws-java-sdk-codecommit:1.12.429')
         
     testImplementation(testFixtures(project(":nextflow")))
diff --git a/plugins/nf-codecommit/changelog.txt b/plugins/nf-codecommit/changelog.txt
index 9c732b5bdd..19bcc2f655 100644
--- a/plugins/nf-codecommit/changelog.txt
+++ b/plugins/nf-codecommit/changelog.txt
@@ -1,5 +1,9 @@
 nf-amazon changelog
 ===================
+0.1.6 - 24 Nov 2023
+- Fix security vulnerabilities (#4513) [a310c777]
+- Bump javax.xml.bind:jaxb-api:2.4.0-b180830.0359
+
 0.1.5 - 15 May 2023
 - Update logging libraries [d7eae86e]
 
diff --git a/plugins/nf-codecommit/src/resources/META-INF/MANIFEST.MF b/plugins/nf-codecommit/src/resources/META-INF/MANIFEST.MF
index c7b679a478..1c15084500 100644
--- a/plugins/nf-codecommit/src/resources/META-INF/MANIFEST.MF
+++ b/plugins/nf-codecommit/src/resources/META-INF/MANIFEST.MF
@@ -1,6 +1,6 @@
 Manifest-Version: 1.0
 Plugin-Class: nextflow.cloud.aws.codecommit.AwsCodeCommitPlugin
 Plugin-Id: nf-codecommit
-Plugin-Version: 0.1.5
+Plugin-Version: 0.1.6
 Plugin-Provider: Seqera Labs
-Plugin-Requires: >=22.06.1-edge
+Plugin-Requires: >=23.11.0-edge
diff --git a/plugins/nf-console/build.gradle b/plugins/nf-console/build.gradle
index 9b88a1daf2..33d5bc1add 100644
--- a/plugins/nf-console/build.gradle
+++ b/plugins/nf-console/build.gradle
@@ -35,7 +35,7 @@ configurations {
 dependencies {
     compileOnly project(':nextflow')
     compileOnly 'org.slf4j:slf4j-api:2.0.7'
-    compileOnly 'org.pf4j:pf4j:3.4.1'
+    compileOnly 'org.pf4j:pf4j:3.10.0'
 
     api("org.codehaus.groovy:groovy-console:3.0.19") { transitive=false }
     api("org.codehaus.groovy:groovy-swing:3.0.19")  { transitive=false }
diff --git a/plugins/nf-console/src/resources/META-INF/MANIFEST.MF b/plugins/nf-console/src/resources/META-INF/MANIFEST.MF
index c29012ba0b..e8ee9c2869 100644
--- a/plugins/nf-console/src/resources/META-INF/MANIFEST.MF
+++ b/plugins/nf-console/src/resources/META-INF/MANIFEST.MF
@@ -1,6 +1,6 @@
 Manifest-Version: 1.0
 Plugin-Class: nextflow.ui.console.ConsolePlugin
 Plugin-Id: nf-console
-Plugin-Version: 1.0.6
+Plugin-Version: 1.0.7
 Plugin-Provider: Seqera Labs
-Plugin-Requires: >=23.04.0
+Plugin-Requires: >=23.11.0-edge
diff --git a/plugins/nf-ga4gh/build.gradle b/plugins/nf-ga4gh/build.gradle
index 1f87616d4d..6767558fe7 100644
--- a/plugins/nf-ga4gh/build.gradle
+++ b/plugins/nf-ga4gh/build.gradle
@@ -35,7 +35,7 @@ configurations {
 dependencies {
     compileOnly project(':nextflow')
     compileOnly 'org.slf4j:slf4j-api:2.0.7'
-    compileOnly 'org.pf4j:pf4j:3.4.1'
+    compileOnly 'org.pf4j:pf4j:3.10.0'
 
     api 'javax.annotation:javax.annotation-api:1.3.2'
     api 'io.swagger:swagger-annotations:1.5.15'
diff --git a/plugins/nf-ga4gh/src/resources/META-INF/MANIFEST.MF b/plugins/nf-ga4gh/src/resources/META-INF/MANIFEST.MF
index 2835d79c84..69568da5ea 100644
--- a/plugins/nf-ga4gh/src/resources/META-INF/MANIFEST.MF
+++ b/plugins/nf-ga4gh/src/resources/META-INF/MANIFEST.MF
@@ -1,6 +1,6 @@
 Manifest-Version: 1.0
 Plugin-Class: nextflow.ga4gh.Ga4ghPlugin
 Plugin-Id: nf-ga4gh
-Plugin-Version: 1.1.0
+Plugin-Version: 1.1.1
 Plugin-Provider: Seqera Labs
-Plugin-Requires: >=23.05.0-edge
+Plugin-Requires: >=23.11.0-edge
diff --git a/plugins/nf-google/build.gradle b/plugins/nf-google/build.gradle
index b1c9178d6f..a9b51fe048 100644
--- a/plugins/nf-google/build.gradle
+++ b/plugins/nf-google/build.gradle
@@ -35,7 +35,7 @@ configurations {
 dependencies {
     compileOnly project(':nextflow')
     compileOnly 'org.slf4j:slf4j-api:2.0.7'
-    compileOnly 'org.pf4j:pf4j:3.4.1'
+    compileOnly 'org.pf4j:pf4j:3.10.0'
 
     api 'com.google.apis:google-api-services-lifesciences:v2beta-rev20210527-1.31.5'
     api 'com.google.auth:google-auth-library-oauth2-http:0.18.0'
diff --git a/plugins/nf-google/changelog.txt b/plugins/nf-google/changelog.txt
index 1637822ae4..30440b654a 100644
--- a/plugins/nf-google/changelog.txt
+++ b/plugins/nf-google/changelog.txt
@@ -1,5 +1,14 @@
 nf-google changelog
 ===================
+1.9.0 - 24 Nov 2023
+- Add labels field in Job request for Google Batch (#4538) [627c595e]
+- Add Google Batch native retry on spot termination (#4500) [ea1c1b70]
+- Add ability detect Google Batch spot interruption (#4462) [d49f02ae]
+- Add Retry policy to Google Storage (#4524) [c271bb18]
+- Fix security vulnerabilities (#4513) [a310c777]
+- Fix Bypass Google Batch Price query if task cpus and memory are defined (#4521) [7f8f20d3]
+- Update logging filter for Google Batch provider. (#4488) [66a3ed19]
+
 1.8.3 - 10 Oct 2023
 - Add setting to enable the use of sync command [f0d5cc5c]
 - Fix Google Batch do not stop running jobs (#4381) [3d6b7358]
diff --git a/plugins/nf-google/src/main/nextflow/cloud/google/GoogleOpts.groovy b/plugins/nf-google/src/main/nextflow/cloud/google/GoogleOpts.groovy
index 3250870ff6..99de01e499 100644
--- a/plugins/nf-google/src/main/nextflow/cloud/google/GoogleOpts.groovy
+++ b/plugins/nf-google/src/main/nextflow/cloud/google/GoogleOpts.groovy
@@ -24,6 +24,7 @@ import groovy.transform.Memoized
 import groovy.transform.ToString
 import groovy.util.logging.Slf4j
 import nextflow.Session
+import nextflow.cloud.google.config.GoogleStorageOpts
 import nextflow.exception.AbortOperationException
 import nextflow.util.Duration
 /**
@@ -46,6 +47,7 @@ class GoogleOpts {
     private boolean enableRequesterPaysBuckets
     private Duration httpConnectTimeout
     private Duration httpReadTimeout
+    private GoogleStorageOpts storageOpts
 
     String getProjectId() { projectId }
     File getCredsFile() { credsFile }
@@ -53,6 +55,16 @@ class GoogleOpts {
     boolean getEnableRequesterPaysBuckets() { enableRequesterPaysBuckets }
     Duration getHttpConnectTimeout() { httpConnectTimeout }
     Duration getHttpReadTimeout() { httpReadTimeout }
+    GoogleStorageOpts getStorageOpts() { storageOpts }
+
+    GoogleOpts(Map opts) {
+        projectId = opts.project as String
+        location = opts.location as String
+        enableRequesterPaysBuckets = opts.enableRequesterPaysBuckets as boolean
+        httpConnectTimeout = opts.httpConnectTimeout ? opts.httpConnectTimeout as Duration : Duration.of('60s')
+        httpReadTimeout = opts.httpReadTimeout ? opts.httpReadTimeout as Duration : Duration.of('60s')
+        storageOpts = new GoogleStorageOpts( opts.storage as Map ?: Map.of() )
+    }
 
     @Memoized
     static GoogleOpts fromSession(Session session) {
@@ -66,12 +78,7 @@ class GoogleOpts {
     }
 
     protected static GoogleOpts fromSession0(Map config) {
-        final result = new GoogleOpts()
-        result.projectId = config.navigate("google.project") as String
-        result.location = config.navigate("google.location") as String
-        result.enableRequesterPaysBuckets = config.navigate('google.enableRequesterPaysBuckets') as boolean
-        result.httpConnectTimeout = config.navigate('google.httpConnectTimeout', '60s') as Duration
-        result.httpReadTimeout = config.navigate('google.httpReadTimeout', '60s') as Duration
+        final result = new GoogleOpts( config.google as Map ?: Map.of() )
 
         if( result.enableRequesterPaysBuckets && !result.projectId )
             throw new IllegalArgumentException("Config option 'google.enableRequesterPaysBuckets' cannot be honoured because the Google project Id has not been specified - Provide it by adding the option 'google.project' in the nextflow.config file")
diff --git a/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchMachineTypeSelector.groovy b/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchMachineTypeSelector.groovy
index 2e21d54042..61b5c4cb8a 100644
--- a/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchMachineTypeSelector.groovy
+++ b/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchMachineTypeSelector.groovy
@@ -90,7 +90,6 @@ class GoogleBatchMachineTypeSelector {
     }
 
     MachineType bestMachineType(int cpus, int memoryMB, String region, boolean spot, boolean fusionEnabled, List families) {
-        final machineTypes = getAvailableMachineTypes(region, spot)
         if (families == null)
             families = Collections.emptyList()
 
@@ -100,7 +99,7 @@ class GoogleBatchMachineTypeSelector {
             if (familyOrType.contains("custom-"))
                 return new MachineType(type: familyOrType, family: 'custom', cpusPerVm: cpus, memPerVm: memoryMB, location: region, priceModel: spot ? PriceModel.spot : PriceModel.standard)
 
-            final machineType = machineTypes.find { it.type == familyOrType }
+            final machineType = getAvailableMachineTypes(region, spot).find { it.type == familyOrType }
             if( machineType )
                 return machineType
         }
@@ -117,7 +116,7 @@ class GoogleBatchMachineTypeSelector {
         final matchMachineType = {String type -> !families || families.find { matchType(it, type) }}
 
         // find machines with enough resources and SSD local disk
-        final validMachineTypes = machineTypes.findAll {
+        final validMachineTypes = getAvailableMachineTypes(region, spot).findAll {
                     it.cpusPerVm >= cpus &&
                     it.memPerVm >= memoryGB &&
                     matchMachineType(it.type)
diff --git a/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchTaskHandler.groovy b/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchTaskHandler.groovy
index 7bd4e1367f..c7cbd17dc6 100644
--- a/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchTaskHandler.groovy
+++ b/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchTaskHandler.groovy
@@ -24,6 +24,7 @@ import com.google.cloud.batch.v1.AllocationPolicy
 import com.google.cloud.batch.v1.ComputeResource
 import com.google.cloud.batch.v1.Environment
 import com.google.cloud.batch.v1.Job
+import com.google.cloud.batch.v1.LifecyclePolicy
 import com.google.cloud.batch.v1.LogsPolicy
 import com.google.cloud.batch.v1.Runnable
 import com.google.cloud.batch.v1.ServiceAccount
@@ -207,6 +208,23 @@ class GoogleBatchTaskHandler extends TaskHandler implements FusionAwareTask {
             )
             .addAllVolumes( launcher.getVolumes() )
 
+        // retry on spot reclaim
+        if( executor.config.maxSpotAttempts ) {
+            // Note: Google Batch uses the special exit status 50001 to signal
+            // the execution was terminated due a spot reclaim. When this happens
+            // The policy re-execute the jobs automatically up to `maxSpotAttempts` times
+            taskSpec
+                .setMaxRetryCount( executor.config.maxSpotAttempts )
+                .addLifecyclePolicies(
+                    LifecyclePolicy.newBuilder()
+                        .setActionCondition(
+                            LifecyclePolicy.ActionCondition.newBuilder()
+                                .addExitCodes(50001)
+                        )
+                        .setAction(LifecyclePolicy.Action.RETRY_TASK)
+                )
+        }
+
         // instance policy
         final allocationPolicy = AllocationPolicy.newBuilder()
         final instancePolicyOrTemplate = AllocationPolicy.InstancePolicyOrTemplate.newBuilder()
@@ -329,6 +347,7 @@ class GoogleBatchTaskHandler extends TaskHandler implements FusionAwareTask {
                 LogsPolicy.newBuilder()
                     .setDestination(LogsPolicy.Destination.CLOUD_LOGGING)
             )
+            .putAllLabels(task.config.getResourceLabels())
             .build()
     }
 
diff --git a/plugins/nf-google/src/main/nextflow/cloud/google/batch/client/BatchConfig.groovy b/plugins/nf-google/src/main/nextflow/cloud/google/batch/client/BatchConfig.groovy
index dae9b35805..a1dcc07a1e 100644
--- a/plugins/nf-google/src/main/nextflow/cloud/google/batch/client/BatchConfig.groovy
+++ b/plugins/nf-google/src/main/nextflow/cloud/google/batch/client/BatchConfig.groovy
@@ -37,8 +37,9 @@ class BatchConfig {
     private List allowedLocations
     private MemoryUnit bootDiskSize
     private String cpuPlatform
-    private boolean spot
+    private int maxSpotAttempts
     private boolean preemptible
+    private boolean spot
     private boolean usePrivateAddress
     private String network
     private String subnetwork
@@ -49,6 +50,7 @@ class BatchConfig {
     List getAllowedLocations() { allowedLocations }
     MemoryUnit getBootDiskSize() { bootDiskSize }
     String getCpuPlatform() { cpuPlatform }
+    int getMaxSpotAttempts() { maxSpotAttempts }
     boolean getPreemptible() { preemptible }
     boolean getSpot() { spot }
     boolean getUsePrivateAddress() { usePrivateAddress }
@@ -63,8 +65,9 @@ class BatchConfig {
         result.allowedLocations = session.config.navigate('google.batch.allowedLocations', List.of()) as List
         result.bootDiskSize = session.config.navigate('google.batch.bootDiskSize') as MemoryUnit
         result.cpuPlatform = session.config.navigate('google.batch.cpuPlatform')
-        result.spot = session.config.navigate('google.batch.spot',false)
+        result.maxSpotAttempts = session.config.navigate('google.batch.maxSpotAttempts',5) as int
         result.preemptible = session.config.navigate('google.batch.preemptible',false)
+        result.spot = session.config.navigate('google.batch.spot',false)
         result.usePrivateAddress = session.config.navigate('google.batch.usePrivateAddress',false)
         result.network = session.config.navigate('google.batch.network')
         result.subnetwork = session.config.navigate('google.batch.subnetwork')
diff --git a/plugins/nf-google/src/main/nextflow/cloud/google/config/GoogleRetryOpts.groovy b/plugins/nf-google/src/main/nextflow/cloud/google/config/GoogleRetryOpts.groovy
new file mode 100644
index 0000000000..de071069c0
--- /dev/null
+++ b/plugins/nf-google/src/main/nextflow/cloud/google/config/GoogleRetryOpts.groovy
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2013-2023, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package nextflow.cloud.google.config
+
+import groovy.transform.CompileStatic
+import nextflow.util.Duration
+
+/**
+ * Model Google storage retry settings
+ *
+ * @author Paolo Di Tommaso 
+ */
+@CompileStatic
+class GoogleRetryOpts {
+
+    final int maxAttempts
+    final double multiplier
+    final Duration maxDelay
+
+    GoogleRetryOpts(Map opts) {
+        maxAttempts = opts.maxAttempts ? opts.maxAttempts as int : 10
+        multiplier = opts.multiplier ? opts.multiplier as double : 2d
+        maxDelay = opts.maxDelay ? opts.maxDelay as Duration : Duration.of('90s')
+    }
+
+    long maxDelaySecs() {
+        return maxDelay.seconds
+    }
+}
diff --git a/plugins/nf-google/src/main/nextflow/cloud/google/config/GoogleStorageOpts.groovy b/plugins/nf-google/src/main/nextflow/cloud/google/config/GoogleStorageOpts.groovy
new file mode 100644
index 0000000000..80a8c8a580
--- /dev/null
+++ b/plugins/nf-google/src/main/nextflow/cloud/google/config/GoogleStorageOpts.groovy
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2013-2023, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package nextflow.cloud.google.config
+
+/**
+ *
+ * @author Paolo Di Tommaso 
+ */
+class GoogleStorageOpts {
+
+    final GoogleRetryOpts retryPolicy
+
+    GoogleStorageOpts(Map opts) {
+        retryPolicy = new GoogleRetryOpts( opts.retryPolicy as Map ?: Map.of()  )
+    }
+
+}
diff --git a/plugins/nf-google/src/main/nextflow/cloud/google/util/GsPathFactory.groovy b/plugins/nf-google/src/main/nextflow/cloud/google/util/GsPathFactory.groovy
index f550d404e7..1924432e4a 100644
--- a/plugins/nf-google/src/main/nextflow/cloud/google/util/GsPathFactory.groovy
+++ b/plugins/nf-google/src/main/nextflow/cloud/google/util/GsPathFactory.groovy
@@ -18,6 +18,7 @@ package nextflow.cloud.google.util
 
 import java.nio.file.Path
 
+import com.google.api.gax.retrying.RetrySettings
 import com.google.cloud.storage.StorageOptions
 import com.google.cloud.storage.contrib.nio.CloudStorageConfiguration
 import com.google.cloud.storage.contrib.nio.CloudStorageFileSystem
@@ -57,15 +58,25 @@ class GsPathFactory extends FileSystemPathFactory {
         return builder.build()
     }
 
-    static protected StorageOptions getCloudStorageOptions(GoogleOpts googleOpts) {
+    static protected StorageOptions getCloudStorageOptions(GoogleOpts opts) {
         final transportOptions = StorageOptions.getDefaultHttpTransportOptions().toBuilder()
-        if( googleOpts.httpConnectTimeout )
-            transportOptions.setConnectTimeout( (int)googleOpts.httpConnectTimeout.toMillis() )
-        if( googleOpts.httpReadTimeout )
-            transportOptions.setReadTimeout( (int)googleOpts.httpReadTimeout.toMillis() )
+        if( opts.httpConnectTimeout )
+            transportOptions.setConnectTimeout( (int)opts.httpConnectTimeout.toMillis() )
+        if( opts.httpReadTimeout )
+            transportOptions.setReadTimeout( (int)opts.httpReadTimeout.toMillis() )
 
-        return StorageOptions.getDefaultInstance().toBuilder()
+        RetrySettings retrySettings =
+            StorageOptions.getDefaultRetrySettings()
+                .toBuilder()
+                .setMaxAttempts(opts.storageOpts.retryPolicy.maxAttempts)
+                .setRetryDelayMultiplier(opts.storageOpts.retryPolicy.multiplier)
+                .setTotalTimeout(org.threeten.bp.Duration.ofSeconds(opts.storageOpts.retryPolicy.maxDelaySecs()))
+                .build()
+
+        return StorageOptions.getDefaultInstance()
+            .toBuilder()
             .setTransportOptions(transportOptions.build())
+            .setRetrySettings(retrySettings)
             .build()
     }
 
diff --git a/plugins/nf-google/src/resources/META-INF/MANIFEST.MF b/plugins/nf-google/src/resources/META-INF/MANIFEST.MF
index 8522585c8e..aa04d212ae 100644
--- a/plugins/nf-google/src/resources/META-INF/MANIFEST.MF
+++ b/plugins/nf-google/src/resources/META-INF/MANIFEST.MF
@@ -1,6 +1,6 @@
 Manifest-Version: 1.0
 Plugin-Class: nextflow.cloud.google.GoogleCloudPlugin
 Plugin-Id: nf-google
-Plugin-Version: 1.8.3
+Plugin-Version: 1.9.0
 Plugin-Provider: Seqera Labs
-Plugin-Requires: >=23.02.0-edge
+Plugin-Requires: >=23.11.0-edge
diff --git a/plugins/nf-google/src/test/nextflow/cloud/google/batch/GoogleBatchTaskHandlerTest.groovy b/plugins/nf-google/src/test/nextflow/cloud/google/batch/GoogleBatchTaskHandlerTest.groovy
index 4d0d36c6f1..ec648112cb 100644
--- a/plugins/nf-google/src/test/nextflow/cloud/google/batch/GoogleBatchTaskHandlerTest.groovy
+++ b/plugins/nf-google/src/test/nextflow/cloud/google/batch/GoogleBatchTaskHandlerTest.groovy
@@ -130,6 +130,7 @@ class GoogleBatchTaskHandlerTest extends Specification {
                 getAllowedLocations() >> ['zones/us-central1-a', 'zones/us-central1-c']
                 getBootDiskSize() >> BOOT_DISK
                 getCpuPlatform() >> CPU_PLATFORM
+                getMaxSpotAttempts() >> 5
                 getSpot() >> true
                 getNetwork() >> 'net-1'
                 getServiceAccountEmail() >> 'foo@bar.baz'
@@ -170,16 +171,20 @@ class GoogleBatchTaskHandlerTest extends Specification {
 
         and:
         def taskGroup = req.getTaskGroups(0)
-        def runnable = taskGroup.getTaskSpec().getRunnables(0)
+        def taskSpec = taskGroup.getTaskSpec()
+        def runnable = taskSpec.getRunnables(0)
         def allocationPolicy = req.getAllocationPolicy()
         def instancePolicy = allocationPolicy.getInstances(0).getPolicy()
         def networkInterface = allocationPolicy.getNetwork().getNetworkInterfaces(0)
         and:
-        taskGroup.getTaskSpec().getComputeResource().getBootDiskMib() == BOOT_DISK.toMega()
-        taskGroup.getTaskSpec().getComputeResource().getCpuMilli() == CPUS * 1_000
-        taskGroup.getTaskSpec().getComputeResource().getMemoryMib() == MEM.toMega()
-        taskGroup.getTaskSpec().getMaxRunDuration().getSeconds() == TIMEOUT.seconds
-        taskGroup.getTaskSpec().getVolumes(0).getMountPath() == '/tmp'
+        taskSpec.getComputeResource().getBootDiskMib() == BOOT_DISK.toMega()
+        taskSpec.getComputeResource().getCpuMilli() == CPUS * 1_000
+        taskSpec.getComputeResource().getMemoryMib() == MEM.toMega()
+        taskSpec.getMaxRunDuration().getSeconds() == TIMEOUT.seconds
+        taskSpec.getVolumes(0).getMountPath() == '/tmp'
+        taskSpec.getMaxRetryCount() == 5
+        taskSpec.getLifecyclePolicies(0).getActionCondition().getExitCodes(0) == 50001
+        taskSpec.getLifecyclePolicies(0).getAction().toString() == 'RETRY_TASK'
         and:
         runnable.getContainer().getCommandsList().join(' ') == '/bin/bash -o pipefail -c bash .command.run'
         runnable.getContainer().getImageUri() == CONTAINER_IMAGE
@@ -212,6 +217,9 @@ class GoogleBatchTaskHandlerTest extends Specification {
         networkInterface.getNoExternalIpAddress() == true
         and:
         req.getLogsPolicy().getDestination().toString() == 'CLOUD_LOGGING'
+        and:
+        req.getLabelsMap() == [foo: 'bar']
+
 
         when:
         req = handler.newSubmitRequest(task, launcher)
diff --git a/plugins/nf-wave/src/test/io/seqera/wave/plugin/WaveObserverTest.groovy b/plugins/nf-google/src/test/nextflow/cloud/google/config/GoogleRetryOptsTest.groovy
similarity index 51%
rename from plugins/nf-wave/src/test/io/seqera/wave/plugin/WaveObserverTest.groovy
rename to plugins/nf-google/src/test/nextflow/cloud/google/config/GoogleRetryOptsTest.groovy
index 69be66d69e..69685472c8 100644
--- a/plugins/nf-wave/src/test/io/seqera/wave/plugin/WaveObserverTest.groovy
+++ b/plugins/nf-google/src/test/nextflow/cloud/google/config/GoogleRetryOptsTest.groovy
@@ -15,33 +15,30 @@
  *
  */
 
-package io.seqera.wave.plugin
+package nextflow.cloud.google.config
 
-import nextflow.Session
+import nextflow.util.Duration
 import spock.lang.Specification
 
 /**
  *
  * @author Paolo Di Tommaso 
  */
-class WaveObserverTest extends Specification {
+class GoogleRetryOptsTest extends Specification {
 
-    def 'should render containers config' () {
-        given:
-        def sess = Mock(Session) {getConfig() >> [:] }
-        def observer = new WaveObserver(sess)
-        and:
-        Map containers = [:]
-        containers.foo = 'quay.io/ubuntu:latest'
-        containers.bar = 'quay.io/alpine:latest'
+    def 'should get retry opts' () {
+        when:
+        def opts1 = new GoogleRetryOpts([:])
+        then:
+        opts1.maxAttempts == 10
+        opts1.multiplier == 2.0d
+        opts1.maxDelay ==  Duration.of('90s')
 
         when:
-        def result = observer.renderContainersConfig(containers)
+        def opts2 = new GoogleRetryOpts([maxAttempts: 5, maxDelay: '5s', multiplier: 10])
         then:
-        result == '''\
-            process { withName: 'foo' { container='quay.io/ubuntu:latest' }}
-            process { withName: 'bar' { container='quay.io/alpine:latest' }}
-            '''.stripIndent(true)
+        opts2.maxAttempts == 5
+        opts2.multiplier == 10d
+        opts2.maxDelay ==  Duration.of('5s')
     }
-
 }
diff --git a/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesHelperTest.groovy b/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesHelperTest.groovy
index b66ee3d6d2..1b0eb7cbe6 100644
--- a/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesHelperTest.groovy
+++ b/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesHelperTest.groovy
@@ -440,10 +440,8 @@ class GoogleLifeSciencesHelperTest extends GoogleSpecification {
 
     def 'should create pipeline actions with keepalive' () {
         given:
-        def helper = Spy(GoogleLifeSciencesHelper)
-        helper.config = Mock(GoogleLifeSciencesConfig) {
-            getKeepAliveOnFailure() >> true
-        }
+        def config = new GoogleLifeSciencesConfig(keepAliveOnFailure: true)
+        def helper = Spy(new GoogleLifeSciencesHelper(config: config))
         and:
         def req = Mock(GoogleLifeSciencesSubmitRequest)
 
diff --git a/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesTaskHandlerTest.groovy b/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesTaskHandlerTest.groovy
index 72d3b8eedf..73ea282be4 100644
--- a/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesTaskHandlerTest.groovy
+++ b/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesTaskHandlerTest.groovy
@@ -198,19 +198,19 @@ class GoogleLifeSciencesTaskHandlerTest extends GoogleSpecification {
         given:
         def workDir = mockGsPath('gs://my-bucket/work/dir')
         and:
+        def config = new GoogleLifeSciencesConfig(
+            project: 'my-project',
+            zones: ['my-zone'],
+            regions: ['my-region'],
+            preemptible: true,
+            bootDiskSize: MemoryUnit.of('20 GB'),
+            usePrivateAddress: true,
+            cpuPlatform: 'Intel Skylake'
+        )
+        and:
         def executor = Mock(GoogleLifeSciencesExecutor) {
             getHelper() >> Mock(GoogleLifeSciencesHelper)
-            getConfig() >> {
-                Mock(GoogleLifeSciencesConfig) {
-                    getProject() >> 'my-project'
-                    getZones() >> ['my-zone']
-                    getRegions() >> ['my-region']
-                    getPreemptible() >> true
-                    getBootDiskSize() >> MemoryUnit.of('20 GB')
-                    getUsePrivateAddress() >> true
-                    getCpuPlatform() >> 'Intel Skylake'
-                }
-            }
+            getConfig() >> config
         }
         and:
         def task = Mock(TaskRun)
diff --git a/plugins/nf-google/src/test/nextflow/cloud/google/util/GsPathFactoryTest.groovy b/plugins/nf-google/src/test/nextflow/cloud/google/util/GsPathFactoryTest.groovy
index e00bf8bd99..8dd6bde8f4 100644
--- a/plugins/nf-google/src/test/nextflow/cloud/google/util/GsPathFactoryTest.groovy
+++ b/plugins/nf-google/src/test/nextflow/cloud/google/util/GsPathFactoryTest.groovy
@@ -20,6 +20,7 @@ import com.google.cloud.storage.StorageOptions
 import nextflow.Global
 import nextflow.Session
 import nextflow.cloud.google.GoogleOpts
+import nextflow.cloud.google.config.GoogleRetryOpts
 import spock.lang.Specification
 import spock.lang.Unroll
 
@@ -87,6 +88,14 @@ class GsPathFactoryTest extends Specification {
             getConfig() >> [google:[httpConnectTimeout: CONNECT, httpReadTimeout: READ]]
         }
         and:
+        def policy = new GoogleRetryOpts([:])
+        def retrySettings = StorageOptions.getDefaultRetrySettings()
+            .toBuilder()
+            .setMaxAttempts(policy.maxAttempts)
+            .setRetryDelayMultiplier(policy.multiplier)
+            .setTotalTimeout(org.threeten.bp.Duration.ofSeconds(policy.maxDelaySecs()))
+            .build()
+        and:
         def opts = GoogleOpts.fromSession(session)
         and:
         def storageOptions = GsPathFactory.getCloudStorageOptions(opts)
@@ -98,6 +107,7 @@ class GsPathFactoryTest extends Specification {
         expect:
         storageOptions == StorageOptions.getDefaultInstance().toBuilder()
             .setTransportOptions(transportOptions.build())
+            .setRetrySettings(retrySettings)
             .build()
 
         where:
@@ -106,4 +116,19 @@ class GsPathFactoryTest extends Specification {
         '30s'   | 30000          | '30s' | 30000
         '60s'   | 60000          | '60s' | 60000
     }
+
+    def 'should apply retry settings' () {
+        given:
+        def session = Mock(Session) {
+            getConfig() >> [google:[storage:[retryPolicy: [maxAttempts: 5, maxDelay:'50s', multiplier: 500]]]]
+        }
+
+        when:
+        def opts = GoogleOpts.fromSession(session)
+        then:
+        opts.storageOpts.retryPolicy.maxAttempts == 5
+        opts.storageOpts.retryPolicy.maxDelaySecs() == 50
+        opts.storageOpts.retryPolicy.multiplier == 500d
+
+    }
 }
diff --git a/plugins/nf-tower/build.gradle b/plugins/nf-tower/build.gradle
index bb5980c1a8..e32f582ede 100644
--- a/plugins/nf-tower/build.gradle
+++ b/plugins/nf-tower/build.gradle
@@ -31,7 +31,7 @@ configurations {
 dependencies {
     compileOnly project(':nextflow')
     compileOnly 'org.slf4j:slf4j-api:2.0.7'
-    compileOnly 'org.pf4j:pf4j:3.4.1'
+    compileOnly 'org.pf4j:pf4j:3.10.0'
 
     api "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.15.0"
     api "com.fasterxml.jackson.core:jackson-databind:2.12.7.1"
diff --git a/plugins/nf-tower/changelog.txt b/plugins/nf-tower/changelog.txt
index 897aa3f12a..9bf2dedd72 100644
--- a/plugins/nf-tower/changelog.txt
+++ b/plugins/nf-tower/changelog.txt
@@ -1,5 +1,9 @@
 nf-tower changelog
 ===================
+1.7.0 - 24 Nov 2023
+- Fix security vulnerabilities (#4513) [a310c777]
+- Remove deprecated TowerArchiver feature [ff8e06a3]
+
 1.6.3 - 10 Oct 2023
 - Add -cloudcache CLI option (#4385) [73fda582]
 
diff --git a/plugins/nf-tower/src/resources/META-INF/MANIFEST.MF b/plugins/nf-tower/src/resources/META-INF/MANIFEST.MF
index 58bd2aa7d9..6522d288d7 100644
--- a/plugins/nf-tower/src/resources/META-INF/MANIFEST.MF
+++ b/plugins/nf-tower/src/resources/META-INF/MANIFEST.MF
@@ -1,6 +1,6 @@
 Manifest-Version: 1.0
 Plugin-Class: io.seqera.tower.plugin.TowerPlugin
 Plugin-Id: nf-tower
-Plugin-Version: 1.6.3
+Plugin-Version: 1.7.0
 Plugin-Provider: Seqera Labs
-Plugin-Requires: >=23.05.0-edge
+Plugin-Requires: >=23.11.0-edge
diff --git a/plugins/nf-wave/build.gradle b/plugins/nf-wave/build.gradle
index 93d749fba3..ed170126a5 100644
--- a/plugins/nf-wave/build.gradle
+++ b/plugins/nf-wave/build.gradle
@@ -31,7 +31,7 @@ configurations {
 dependencies {
     compileOnly project(':nextflow')
     compileOnly 'org.slf4j:slf4j-api:2.0.7'
-    compileOnly 'org.pf4j:pf4j:3.4.1'
+    compileOnly 'org.pf4j:pf4j:3.10.0'
     api 'org.apache.commons:commons-compress:1.21'
     api 'org.apache.commons:commons-lang3:3.12.0'
     api 'com.google.code.gson:gson:2.10.1'
diff --git a/plugins/nf-wave/changelog.txt b/plugins/nf-wave/changelog.txt
index 5886b9be8a..eb3a22bf38 100644
--- a/plugins/nf-wave/changelog.txt
+++ b/plugins/nf-wave/changelog.txt
@@ -1,5 +1,14 @@
 nf-wave changelog
 ==================
+1.1.0 - 24 Nov 2023
+- Add Retry policy to Google Storage (#4524) [c271bb18]
+- Add support for Singularity OCI mode (#4440) [f5362a7b]
+- Fix detection of Conda local path made by Wave client (#4532) [4d5bc216]
+- Fix security vulnerabilities (#4513) [a310c777]
+- Fix container hashing for Singularity + Wave containers [4c6f2e85]
+- Use consistently NXF_TASK_WORKDIR (#4484) [48ee3c64]
+- Fix Inspect command fails with Singularity [f5bb829f]
+
 1.0.0 - 15 Oct 2023
 - Fix conda channels order [ci fast] [6672c6d7]
 - Bump nf-wave@1.0.0 [795849d7]
diff --git a/plugins/nf-wave/src/main/io/seqera/wave/plugin/WaveClient.groovy b/plugins/nf-wave/src/main/io/seqera/wave/plugin/WaveClient.groovy
index 7e4b4f7487..367ac8d4be 100644
--- a/plugins/nf-wave/src/main/io/seqera/wave/plugin/WaveClient.groovy
+++ b/plugins/nf-wave/src/main/io/seqera/wave/plugin/WaveClient.groovy
@@ -69,6 +69,9 @@ import org.slf4j.LoggerFactory
 @CompileStatic
 class WaveClient {
 
+    final static public String DEFAULT_S5CMD_AMD64_URL = 'https://nf-xpack.seqera.io/s5cmd/linux_amd64_2.0.0.json'
+    final static public String DEFAULT_S5CMD_ARM64_URL = 'https://nf-xpack.seqera.io/s5cmd/linux_arm64_2.0.0.json'
+
     private static Logger log = LoggerFactory.getLogger(WaveClient)
 
     final static private String[] REQUEST_HEADERS =  new String[]{
@@ -112,11 +115,17 @@ class WaveClient {
 
     final private String waveRegistry
 
+    final private boolean awsFargate
+
+    final private URL s5cmdConfigUrl
+
     WaveClient(Session session) {
         this.session = session
         this.config = new WaveConfig(session.config.wave as Map ?: Collections.emptyMap(), SysEnv.get())
         this.fusion = new FusionConfig(session.config.fusion as Map ?: Collections.emptyMap(), SysEnv.get())
         this.tower = new TowerConfig(session.config.tower as Map ?: Collections.emptyMap(), SysEnv.get())
+        this.awsFargate = WaveFactory.isAwsBatchFargateMode(session.config)
+        this.s5cmdConfigUrl = session.config.navigate('wave.s5cmdConfigUrl') as URL
         this.endpoint = config.endpoint()
         this.condaChannels = session.getCondaConfig()?.getChannels() ?: DEFAULT_CONDA_CHANNELS
         log.debug "Wave config: $config"
@@ -280,12 +289,23 @@ class WaveClient {
                 : new URL(FusionConfig.DEFAULT_FUSION_AMD64_URL)
     }
 
+    protected URL defaultS5cmdUrl(String platform) {
+        final isArm = platform.tokenize('/')?.contains('arm64')
+        return isArm
+            ? new URL(DEFAULT_S5CMD_ARM64_URL)
+            : new URL(DEFAULT_S5CMD_AMD64_URL)
+    }
+
     ContainerConfig resolveContainerConfig(String platform = DEFAULT_DOCKER_PLATFORM) {
         final urls = new ArrayList(config.containerConfigUrl())
         if( fusion.enabled() ) {
             final fusionUrl = fusion.containerConfigUrl() ?: defaultFusionUrl(platform)
             urls.add(fusionUrl)
         }
+        if( awsFargate ) {
+            final s5cmdUrl = s5cmdConfigUrl ?: defaultS5cmdUrl(platform)
+            urls.add(s5cmdUrl)
+        }
         if( !urls )
             return null
         def result = new ContainerConfig()
@@ -567,6 +587,8 @@ class WaveClient {
             return false
         if( value.startsWith('http://') || value.startsWith('https://') )
             return false
+        if( value.startsWith('/') && !value.contains('\n') )
+            return true
         return value.endsWith('.yaml') || value.endsWith('.yml') || value.endsWith('.txt')
     }
 
diff --git a/plugins/nf-wave/src/main/io/seqera/wave/plugin/WaveFactory.groovy b/plugins/nf-wave/src/main/io/seqera/wave/plugin/WaveFactory.groovy
index 48ae1b974e..6e16e301fb 100644
--- a/plugins/nf-wave/src/main/io/seqera/wave/plugin/WaveFactory.groovy
+++ b/plugins/nf-wave/src/main/io/seqera/wave/plugin/WaveFactory.groovy
@@ -46,19 +46,27 @@ class WaveFactory implements TraceObserverFactory {
         }
         
         if( fusion.enabled ) {
-            if( !wave.enabled ) {
-                throw new AbortOperationException("Fusion feature requires enabling Wave service")
-            }
-            else {
-                log.debug "Detected Fusion enabled -- Enabling bundle project resources -- Disabling upload of remote bin directory"
-                wave.bundleProjectResources = true
-                session.disableRemoteBinDir = true
-            }
+            checkWaveRequirement(session, wave, 'Fusion')
         }
+        if( isAwsBatchFargateMode(config) ) {
+            checkWaveRequirement(session, wave, 'Fargate')
+        }
+
+        return List.of()
+    }
+
+    protected void checkWaveRequirement(Session session, Map wave, String feature) {
+        if( !wave.enabled ) {
+            throw new AbortOperationException("$feature feature requires enabling Wave service")
+        }
+        else {
+            log.debug "Detected $feature enabled -- Enabling bundle project resources -- Disabling upload of remote bin directory"
+            wave.bundleProjectResources = true
+            session.disableRemoteBinDir = true
+        }
+    }
 
-        final observer = new WaveObserver(session)
-        return wave.enabled && observer.reportOpts().enabled()
-                ? List.of(observer)
-                : List.of()
+    static boolean isAwsBatchFargateMode(Map config) {
+        return 'fargate'.equalsIgnoreCase(config.navigate('aws.batch.platformType') as String)
     }
 }
diff --git a/plugins/nf-wave/src/main/io/seqera/wave/plugin/WaveObserver.groovy b/plugins/nf-wave/src/main/io/seqera/wave/plugin/WaveObserver.groovy
deleted file mode 100644
index 7e8a3e6e1d..0000000000
--- a/plugins/nf-wave/src/main/io/seqera/wave/plugin/WaveObserver.groovy
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Copyright 2013-2023, Seqera Labs
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package io.seqera.wave.plugin
-
-import java.util.concurrent.ConcurrentHashMap
-
-import groovy.transform.CompileStatic
-import groovy.util.logging.Slf4j
-import io.seqera.wave.plugin.config.ReportOpts
-import nextflow.Session
-import nextflow.file.FileHelper
-import nextflow.processor.TaskHandler
-import nextflow.trace.TraceObserver
-import nextflow.trace.TraceRecord
-/**
- *
- * @author Paolo Di Tommaso 
- */
-@Slf4j
-@CompileStatic
-@Deprecated
-class WaveObserver implements TraceObserver {
-
-    private WaveClient client
-
-    private ConcurrentHashMap containers = new ConcurrentHashMap<>()
-
-    WaveObserver(Session session) {
-        this.client = new WaveClient(session)
-    }
-
-    @Override
-    void onFlowCreate(Session session) {
-        log.warn "Wave report feature has been deprecated in favour of the new 'nextflow inspect' command"
-    }
-
-    protected void apply(TaskHandler handler) {
-        final process = handler.task.getProcessor().getName()
-        containers.computeIfAbsent(process, (String it) -> handler.task.getContainer())
-    }
-
-    void onProcessComplete(TaskHandler handler, TraceRecord trace){
-        apply(handler)
-    }
-
-    void onProcessCached(TaskHandler handler, TraceRecord trace){
-        apply(handler)
-    }
-
-    @Override
-    void onFlowComplete() {
-        final result = renderContainersConfig(containers)
-        // save the report file
-        FileHelper
-                .asPath(reportOpts().file())
-                .text = result.toString()
-    }
-
-    protected String renderContainersConfig(Map containers) {
-        final result = new StringBuilder()
-        for( Map.Entry entry : containers ) {
-            result.append("process { withName: '${entry.key}' { container='$entry.value' }}\n")
-        }
-        return result.toString()
-    }
-
-    ReportOpts reportOpts() {
-        client.config().reportOpts()
-    }
-}
diff --git a/plugins/nf-wave/src/main/io/seqera/wave/plugin/resolver/WaveContainerResolver.groovy b/plugins/nf-wave/src/main/io/seqera/wave/plugin/resolver/WaveContainerResolver.groovy
index 28c64973f3..8049d292da 100644
--- a/plugins/nf-wave/src/main/io/seqera/wave/plugin/resolver/WaveContainerResolver.groovy
+++ b/plugins/nf-wave/src/main/io/seqera/wave/plugin/resolver/WaveContainerResolver.groovy
@@ -41,7 +41,7 @@ import nextflow.processor.TaskRun
 @Priority(-10)  // <-- lower is higher, this is needed to override default provider behavior
 class WaveContainerResolver implements ContainerResolver {
 
-    private ContainerResolver defaultResolver = new DefaultContainerResolver()
+    private DefaultContainerResolver defaultResolver = new DefaultContainerResolver()
     static final private List DOCKER_LIKE = ['docker','podman','sarus']
     static final private List SINGULARITY_LIKE = ['singularity','apptainer']
     static final private String DOCKER_PREFIX = 'docker://'
@@ -93,11 +93,15 @@ class WaveContainerResolver implements ContainerResolver {
             }
             // fetch the wave container name
             final image = waveContainer(task, imageName, singularitySpec)
+            // when wave returns no info, just default to standard behaviour
+            if( !image ) {
+                return defaultResolver.resolveImage(task, imageName)
+            }
             // oras prefixed container are served directly
-            if( image && image.target.startsWith("oras://") )
+            if( image.target.startsWith("oras://") )
                 return image
-            // otherwise adapt it to singularity format
-            return defaultResolver.resolveImage(task, image.target)
+            // otherwise adapt it to singularity format using the target containerInfo to avoid the cache invalidation
+            return defaultResolver.resolveImage(task, image.target, image.hashKey)
         }
         else
             throw new IllegalArgumentException("Wave does not support '$engine' container engine")
diff --git a/plugins/nf-wave/src/resources/META-INF/MANIFEST.MF b/plugins/nf-wave/src/resources/META-INF/MANIFEST.MF
index c45b973508..10bc31538d 100644
--- a/plugins/nf-wave/src/resources/META-INF/MANIFEST.MF
+++ b/plugins/nf-wave/src/resources/META-INF/MANIFEST.MF
@@ -1,6 +1,6 @@
 Manifest-Version: 1.0
 Plugin-Class: io.seqera.wave.plugin.WavePlugin
 Plugin-Id: nf-wave
-Plugin-Version: 1.0.0
+Plugin-Version: 1.1.0
 Plugin-Provider: Seqera Labs
-Plugin-Requires: >=23.05.0-edge
+Plugin-Requires: >=23.11.0-edge
diff --git a/plugins/nf-wave/src/test/io/seqera/wave/plugin/WaveClientTest.groovy b/plugins/nf-wave/src/test/io/seqera/wave/plugin/WaveClientTest.groovy
index 6842819bb2..a15c8cc2bb 100644
--- a/plugins/nf-wave/src/test/io/seqera/wave/plugin/WaveClientTest.groovy
+++ b/plugins/nf-wave/src/test/io/seqera/wave/plugin/WaveClientTest.groovy
@@ -33,7 +33,6 @@ import groovy.util.logging.Slf4j
 import nextflow.Session
 import nextflow.SysEnv
 import nextflow.container.inspect.ContainerInspectMode
-import nextflow.container.inspect.ContainersInspector
 import nextflow.extension.FilesEx
 import nextflow.file.FileHelper
 import nextflow.processor.TaskRun
@@ -1206,6 +1205,25 @@ class WaveClientTest extends Specification {
         'linux/arm64/v8'    | 'https://fusionfs.seqera.io/releases/v2.2-arm64.json'
     }
 
+    @Unroll
+    def 'should get s5cmd default url' () {
+        given:
+        def sess = Mock(Session) {getConfig() >> [:] }
+        and:
+        def wave = Spy(new WaveClient(sess))
+
+        expect:
+        wave.defaultS5cmdUrl(ARCH).toURI().toString() == EXPECTED
+
+        where:
+        ARCH                | EXPECTED
+        'linux/amd64'       | 'https://nf-xpack.seqera.io/s5cmd/linux_amd64_2.0.0.json'
+        'linux/x86_64'      | 'https://nf-xpack.seqera.io/s5cmd/linux_amd64_2.0.0.json'
+        'arm64'             | 'https://nf-xpack.seqera.io/s5cmd/linux_arm64_2.0.0.json'
+        'linux/arm64'       | 'https://nf-xpack.seqera.io/s5cmd/linux_arm64_2.0.0.json'
+        'linux/arm64/v8'    | 'https://nf-xpack.seqera.io/s5cmd/linux_arm64_2.0.0.json'
+    }
+
     def 'should check is local conda file' () {
         expect:
         WaveClient.isCondaLocalFile(CONTENT) == EXPECTED
@@ -1215,6 +1233,7 @@ class WaveClientTest extends Specification {
         'foo'               | false
         'foo.yml'           | true
         'foo.txt'           | true
+        '/foo/bar'          | true
         'foo\nbar.yml'      | false
         'http://foo.com'    | false
     }
diff --git a/plugins/nf-wave/src/test/io/seqera/wave/plugin/WaveFactoryTest.groovy b/plugins/nf-wave/src/test/io/seqera/wave/plugin/WaveFactoryTest.groovy
index 980c9c9c9d..482c24e453 100644
--- a/plugins/nf-wave/src/test/io/seqera/wave/plugin/WaveFactoryTest.groovy
+++ b/plugins/nf-wave/src/test/io/seqera/wave/plugin/WaveFactoryTest.groovy
@@ -49,6 +49,22 @@ class WaveFactoryTest extends Specification {
         [wave:[enabled:true], fusion:[enabled:true]]     | [wave:[enabled:true,bundleProjectResources:true], fusion:[enabled:true]] | 1
     }
 
+    @Unroll
+    def 'should check s5cmd is enabled' () {
+        given:
+        def factory = new WaveFactory()
+
+        expect:
+        factory.isAwsBatchFargateMode(CONFIG) == EXPECTED
+        
+        where:
+        CONFIG                                  | EXPECTED
+        [:]                                     | false
+        [aws:[batch:[platformType:'foo']]]       | false
+        [aws:[batch:[platformType:'fargate']]]   | true
+        [aws:[batch:[platformType:'Fargate']]]   | true
+
+    }
     def 'should fail when wave is disabled' () {
         given:
         def CONFIG = [wave:[:], fusion:[enabled:true]]
diff --git a/plugins/nf-wave/src/test/io/seqera/wave/plugin/resolver/WaveContainerResolverTest.groovy b/plugins/nf-wave/src/test/io/seqera/wave/plugin/resolver/WaveContainerResolverTest.groovy
index 8f46949bf4..74f2db4519 100644
--- a/plugins/nf-wave/src/test/io/seqera/wave/plugin/resolver/WaveContainerResolverTest.groovy
+++ b/plugins/nf-wave/src/test/io/seqera/wave/plugin/resolver/WaveContainerResolverTest.groovy
@@ -67,7 +67,7 @@ class WaveContainerResolverTest extends Specification {
         _ * task.getContainerConfig() >> Mock(ContainerConfig) { getEngine()>>'singularity' }
         and:
         1 * resolver.waveContainer(task, CONTAINER_NAME, false) >> WAVE_CONTAINER
-        1 * defaultResolver.resolveImage(task, WAVE_CONTAINER.target) >> SINGULARITY_CONTAINER
+        1 * defaultResolver.resolveImage(task, WAVE_CONTAINER.target, WAVE_CONTAINER.hashKey) >> SINGULARITY_CONTAINER
         and:
         result == SINGULARITY_CONTAINER
 
diff --git a/tests/checks/fusion-symlink.nf/.checks b/tests/checks/fusion-symlink.nf/.checks
new file mode 100644
index 0000000000..a28366d245
--- /dev/null
+++ b/tests/checks/fusion-symlink.nf/.checks
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Skip test if AWS keys are missing
+if [ -z "$AWS_ACCESS_KEY_ID" ]; then 
+  echo "Missing AWS credentials -- Skipping test"
+  exit 0
+fi
+
+#
+# normal run
+#
+echo initial run
+$NXF_RUN -c .config
+
+$NXF_CMD fs cp s3://nextflow-ci/work/ci-test/fusion-symlink/data.txt data.txt
+cmp data.txt .expected || false
+
+#
+# resume run
+#
+echo resumed run
+$NXF_RUN -c .config -resume
+
+$NXF_CMD fs cp s3://nextflow-ci/work/ci-test/fusion-symlink/data.txt data.txt
+cmp data.txt .expected || false
diff --git a/tests/checks/fusion-symlink.nf/.config b/tests/checks/fusion-symlink.nf/.config
new file mode 100644
index 0000000000..630af142fa
--- /dev/null
+++ b/tests/checks/fusion-symlink.nf/.config
@@ -0,0 +1,4 @@
+workDir = 's3://nextflow-ci/work'
+fusion.enabled = true
+fusion.exportStorageCredentials = true
+wave.enabled = true
\ No newline at end of file
diff --git a/tests/checks/fusion-symlink.nf/.expected b/tests/checks/fusion-symlink.nf/.expected
new file mode 100644
index 0000000000..e427984d4a
--- /dev/null
+++ b/tests/checks/fusion-symlink.nf/.expected
@@ -0,0 +1 @@
+HELLO
diff --git a/tests/checks/topic-channel.nf/.checks b/tests/checks/topic-channel.nf/.checks
new file mode 100644
index 0000000000..425cb15ae2
--- /dev/null
+++ b/tests/checks/topic-channel.nf/.checks
@@ -0,0 +1,15 @@
+#
+# initial run
+#
+echo Initial run
+$NXF_RUN
+
+cmp versions.txt .expected || false
+
+#
+# Resumed run
+#
+echo Resumed run
+$NXF_RUN -resume
+
+cmp versions.txt .expected || false
diff --git a/tests/checks/topic-channel.nf/.expected b/tests/checks/topic-channel.nf/.expected
new file mode 100644
index 0000000000..1bb4d6b958
--- /dev/null
+++ b/tests/checks/topic-channel.nf/.expected
@@ -0,0 +1,2 @@
+bar: 0.9.0
+foo: 0.1.0
diff --git a/tests/fusion-symlink.nf b/tests/fusion-symlink.nf
new file mode 100644
index 0000000000..30d0cee56d
--- /dev/null
+++ b/tests/fusion-symlink.nf
@@ -0,0 +1,61 @@
+#!/usr/bin/env nextflow
+/*
+ * Copyright 2013-2023, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+process CREATE {
+
+    output:
+    path "data.txt"
+
+    script:
+    """
+    echo HELLO > data.txt
+    """
+}
+
+process FORWARD {
+
+    input:
+    path "data.txt"
+
+    output:
+    path "data.txt"
+
+    script:
+    """
+    echo AND
+    """
+}
+
+process PUBLISH {
+    publishDir "s3://nextflow-ci/work/ci-test/fusion-symlink"
+
+    input:
+    path "data.txt"
+
+    output:
+    path "data.txt"
+
+    script:
+    """
+    echo BYE
+    """
+}
+
+workflow {
+    CREATE | FORWARD | PUBLISH
+}
diff --git a/tests/topic-channel.nf b/tests/topic-channel.nf
new file mode 100644
index 0000000000..c5eb17bc97
--- /dev/null
+++ b/tests/topic-channel.nf
@@ -0,0 +1,37 @@
+
+nextflow.preview.topic = true
+
+process foo {
+  input:
+  val(index)
+
+  output:
+  stdout emit: versions, topic: versions
+
+  script:
+  """
+  echo 'foo: 0.1.0'
+  """
+}
+
+process bar {
+  input:
+  val(index)
+
+  output:
+  stdout emit: versions, topic: versions
+
+  script:
+  """
+  echo 'bar: 0.9.0'
+  """
+}
+
+workflow {
+  Channel.of( 1..3 ) | foo
+  Channel.of( 1..3 ) | bar
+
+  Channel.topic('versions')
+  | unique
+  | collectFile(name: 'versions.txt', sort: true, storeDir: '.')
+}
diff --git a/validation/awsbatch.sh b/validation/awsbatch.sh
index 504e0a8b63..6b0298756b 100644
--- a/validation/awsbatch.sh
+++ b/validation/awsbatch.sh
@@ -53,3 +53,10 @@ $NXF_CMD run nextflow-io/rnaseq-nf \
     -resume
 [[ `grep -c 'Using Nextflow cache factory: nextflow.cache.CloudCacheFactory' .nextflow.log` == 1 ]] || false
 [[ `grep -c 'Cached process > ' .nextflow.log` == 4 ]] || false
+
+## run with fargate + wave
+NXF_CLOUDCACHE_PATH=s3://nextflow-ci/cache \
+$NXF_CMD run nextflow-io/rnaseq-nf \
+    -profile batch \
+    -plugins nf-cloudcache,nf-wave \
+    -c awsfargate.config
diff --git a/validation/awsfargate.config b/validation/awsfargate.config
new file mode 100644
index 0000000000..d05ba3a17e
--- /dev/null
+++ b/validation/awsfargate.config
@@ -0,0 +1,14 @@
+/*
+ * do not include plugin requirements otherwise latest
+ * published version will be downloaded instead of using local build
+ */
+
+workDir = 's3://nextflow-ci/work'
+process.executor = 'awsbatch'
+process.queue = 'nextflow-fg'
+process.container = 'quay.io/nextflow/rnaseq-nf:latest'
+aws.region = 'eu-west-1'
+aws.batch.platformType = 'fargate'
+aws.batch.jobRole = 'arn:aws:iam::195996028523:role/nf-batchjobrole'
+aws.batch.executionRole = 'arn:aws:iam::195996028523:role/nf-batchexecutionrole'
+wave.enabled = true
diff --git a/validation/google.sh b/validation/google.sh
index e0f7792870..d97fedd4c1 100644
--- a/validation/google.sh
+++ b/validation/google.sh
@@ -6,16 +6,7 @@ get_abs_filename() {
 
 export NXF_CMD=${NXF_CMD:-$(get_abs_filename ../launch.sh)}
 
-#
-# setup credentials:
-#  1. decrypt credentials file
-#  2. export required env var
-# Note: file was encrypted with the command:
-#  gpg --symmetric --cipher-algo AES256 --output ./google_credentials.gpg $GOOGLE_APPLICATION_CREDENTIALS
-#
-# More details https://help.github.com/en/actions/automating-your-workflow-with-github-actions/creating-and-using-encrypted-secrets
-#   
-gpg --quiet --batch --yes --decrypt --passphrase=$GOOGLE_SECRET --output google_credentials.json ./google_credentials.gpg
+echo $GOOGLE_SECRET | base64 -d > $PWD/google_credentials.json
 export GOOGLE_APPLICATION_CREDENTIALS=$PWD/google_credentials.json
 
 [[ $TOWER_ACCESS_TOKEN ]] && OPTS='-with-tower' || OPTS=''
diff --git a/validation/google_credentials.gpg b/validation/google_credentials.gpg
deleted file mode 100644
index b0dfd31a26150fda8afee59ba09389dc0f353239..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 1726
zcmV;v20{6Z4Fm}T0g0{CpW8S<7j3FZ$gYUPUUD
zMLs+pltI<3&pX+HsP8iE<=Vaygaf!oQ0f6JqxcmjpQA3gfh@Ly%oHtghHeq81b9Su
z*k`B-<{6&*%N_(?UK$q-p({z<&Wnw)RM+ifaZSv&aa;C(v2)DiNsk58ijs=9Sr$2AHu^XM~E(I_av{&JPoUm~*3^(FmujYp8|9
z5b_suQUsz&rgb}oG{Pt4Hh|C9ZlYtSLZAl7VzIYS>xQ)tH&g-&*1Lnpa^IbcYAm03
z2lt*J7%Q=x4}`2qC%~NZUo&7^T`uRLDq-C7W1-VI#!BsJ%#-0140TR|qcwP6%0>BwZ0RvVApB~iHf3DAuM2gB^Qw`tzEh!pdp
zYIkSCMq7}Qqqyi`GqHmu9d3=EvsQs44|;WXCk?e&OOEip0I2y((S-&h?4#Y2bA@c8
z2<-tNIFK31#uxX4S+t429+^YFH;t;ZeR%hT
zA-H<@PEkmkM*cS|PZqOk{MA4F3XxUz{cBq9I)`OYk$f~ik>w|@u(%QVKgrk3`naM3
z+JNKM+g@~;?~QE1Y%=TPMGh%HkIkwNPmsg~KVhK7QBird&;_&9vh9#LW9T~mo@C+w
zQ2qhMwECqSkZ`G6VRV954A`7Fosjm`ld_zm0_dTuA^V%HDBCnu~e8rjwmY!wOsqSf&>quHv2uMhPxBa5l(
zpCP(^Z$wDnboXJ_EJM>NTJN0k@?~bsO=qtt<&KQs(~C+GEu@CNBg0PPC#AvnWUr^N
zccRvVDc6(^Z6x#u%l2NV-_7$M!3!)?s11eMhk+^W#4QH90Zt6b{zJDwAeqv3p&?x`
z49L@!rc0`K>A;NM>U7OT2=K-EdTPMLWlaL&YpkU7wd4pz{)?rdd8h2MAAt+gPE9)`
zW75A}c60#0V*It7zb|=i)*gPoU9YHDsTwJ`1*>YdFW!uqTLjb~JvD)_H942GKyV8%
zt2>e<6;>~t3ka}6?Ub@m#0N(~|Gc#h%;BeWWn)mco-Puw{@1^iKL#YZ$~7AE8Yo`J
z)Lk_T2HLF62&ax){X8n1kioK7>gHF~tOMJ8$$eXQzw?ni>rc!?u--5;D%lr!UM$hX
z#r`6^+(S1&jtxJTFeZ+I9Jp{o<>wSjEI4?YnR!ZKM+`I^O56hdiQyi5`QQzROs>tW!d-(>WQC*fQ9A
z^Wu3_?b&@EB?CErz`UK(yHjF(`-FXVd|;^M(4|fAOMH5=*e+mj2r_j+US7J+HzK6{
zrVKYUN560u<0f}kUL)3NPRS4|bV;);fWWUCyfbtP;q7mR_1PXxeX0q15&?6#CHS4*
zXqQYs$6e$L^c-G0EqNJxko%}}UxXJGvSjja{rDQZqvbNbQ{M04(Ne9;p0Y2&1AeD|$ma*YU!P+W=jj&E*w*UYD


From 48423e4729eb2d90cc9aac1de066a0c470fabdf8 Mon Sep 17 00:00:00 2001
From: Ben Sherman 
Date: Sun, 17 Dec 2023 22:11:38 -0600
Subject: [PATCH 26/36] Fix failing integration tests

Signed-off-by: Ben Sherman 
---
 .../nextflow/extension/CombineManyOp.groovy   | 27 +++++++++++++++++--
 .../processor/TaskFileCollector.groovy        | 11 +++++---
 .../nextflow/processor/TaskProcessor.groovy   |  7 ++---
 .../groovy/nextflow/script/ProcessDef.groovy  | 22 +++++++--------
 .../nextflow/script/ProcessInput.groovy       | 16 +++--------
 .../nextflow/script/ScriptTokens.groovy       |  2 +-
 tests/singleton.nf                            |  2 +-
 7 files changed, 51 insertions(+), 36 deletions(-)

diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/CombineManyOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/CombineManyOp.groovy
index d709dbbc68..324f9e73ed 100644
--- a/modules/nextflow/src/main/groovy/nextflow/extension/CombineManyOp.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/extension/CombineManyOp.groovy
@@ -45,6 +45,8 @@ class CombineManyOp {
 
     private boolean emitSingleton
 
+    private boolean emitCombination
+
     private transient List combinations
 
     CombineManyOp(List sources, List iterators) {
@@ -53,6 +55,7 @@ class CombineManyOp {
         this.queues = sources.collect( ch -> [] )
         this.singletons = sources.collect( ch -> !CH.isChannelQueue(ch) )
         this.emitSingleton = iterators.size() == 0 && singletons.every()
+        this.emitCombination = iterators.size() == sources.size()
     }
 
     private Map handler(int index, DataflowWriteChannel target, AtomicInteger counter) {
@@ -61,7 +64,7 @@ class CombineManyOp {
             onNext(target, index, it)
         }
         opts.onComplete = {
-            if( counter.decrementAndGet() == 0 && !emitSingleton )
+            if( counter.decrementAndGet() == 0 && !emitSingleton && !emitCombination )
                 target.bind(Channel.STOP)
         }
         return opts
@@ -74,6 +77,26 @@ class CombineManyOp {
         if( queues.any(q -> q.size() == 0) )
             return
 
+        // emit singleton value if every source is a singleton
+        if( emitSingleton ) {
+            final args = queues.collect(q -> q.first())
+            target.bind(args)
+            return
+        }
+
+        // emit combinations once if every source is an iterator
+        if( emitCombination ) {
+            emit(target)
+            target.bind(Channel.STOP)
+            return
+        }
+
+        // otherwise emit as many items as are available
+        while( queues.every(q -> q.size() > 0) )
+            emit(target)
+    }
+
+    private void emit(DataflowWriteChannel target) {
         // emit the next item if there are no iterators
         if( iterators.size() == 0 ) {
             final args = (0..
@@ -96,7 +119,7 @@ class CombineManyOp {
             for( int k = 0; k < entries.size(); k++ )
                 args[iterators[k]] = entries[k]
 
-            target.bind(args)
+            target.bind(new ArrayList(args))
         }
     }
 
diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskFileCollector.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskFileCollector.groovy
index b0bacfdceb..2f977f3b70 100644
--- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskFileCollector.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskFileCollector.groovy
@@ -42,14 +42,17 @@ class TaskFileCollecter {
 
     private TaskRun task
 
+    private Path workDir
+
     TaskFileCollecter(ProcessFileOutput param, TaskRun task) {
         this.param = param
         this.task = task
+        this.workDir = task.getTargetDir()
     }
 
     Object collect() {
         final List allFiles = []
-        final filePatterns = param.getFilePatterns(task.context, task.workDir)
+        final filePatterns = param.getFilePatterns(task.context, workDir)
         boolean inputsExcluded = false
 
         for( String filePattern : filePatterns ) {
@@ -57,7 +60,7 @@ class TaskFileCollecter {
 
             final splitter = param.glob ? FilePatternSplitter.glob().parse(filePattern) : null
             if( splitter?.isPattern() ) {
-                result = fetchResultFiles(filePattern, task.workDir)
+                result = fetchResultFiles(filePattern, workDir)
                 if( result && !param.includeInputs ) {
                     result = excludeStagedInputs(task, result)
                     log.trace "Process ${task.lazyName()} > after removing staged inputs: ${result}"
@@ -66,7 +69,7 @@ class TaskFileCollecter {
             }
             else {
                 final path = param.glob ? splitter.strip(filePattern) : filePattern
-                final file = task.workDir.resolve(path)
+                final file = workDir.resolve(path)
                 final exists = checkFileExists(file)
                 if( exists )
                     result = List.of(file)
@@ -132,7 +135,7 @@ class TaskFileCollecter {
 
         for( int i = 0; i < collectedFiles.size(); i++ ) {
             final file = collectedFiles.get(i)
-            final relativeName = task.workDir.relativize(file).toString()
+            final relativeName = workDir.relativize(file).toString()
             if( !allStagedFiles.contains(relativeName) )
                 result.add(file)
         }
diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy
index 08f08546de..7b84c149d9 100644
--- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy
@@ -639,7 +639,7 @@ class TaskProcessor {
         }
 
         // -- when store path is set, only output params of type 'file' can be specified
-        if( task.inputFiles.size() == 0 ) {
+        if( config.getOutputs().getFiles().size() == 0 ) {
             checkWarn "[${safeTaskName(task)}] StoreDir can only be used when using 'file' outputs"
             return false
         }
@@ -1895,8 +1895,9 @@ class TaskProcessor {
         Object controlMessageArrived(final DataflowProcessor processor, final DataflowReadChannel channel, final int index, final Object message) {
             // apparently auto if-guard instrumented by @Slf4j is not honoured in inner classes - add it explicitly
             if( log.isTraceEnabled() ) {
+                def channelName = config.getInputs()?.names?.get(index)
                 def taskName = currentTask.get()?.name ?: name
-                log.trace "<${taskName}> Control message arrived => ${message}"
+                log.trace "<${taskName}> Control message arrived ${channelName} => ${message}"
             }
 
             super.controlMessageArrived(processor, channel, index, message)
@@ -1905,7 +1906,7 @@ class TaskProcessor {
                 // apparently auto if-guard instrumented by @Slf4j is not honoured in inner classes - add it explicitly
                 if( log.isTraceEnabled() )
                     log.trace "<${name}> Poison pill arrived; port: $index"
-                closed.set(true)
+                closed.set(true) // mark the process as closed
                 state.update { StateObj it -> it.poison() }
             }
 
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy
index 1343631cab..4a2321b567 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy
@@ -158,18 +158,16 @@ class ProcessDef extends BindableDef implements IterableDef, ChainableDef {
             declaredInputs[i].bind(args[i])
 
         // combine input channels
-        final inputs = declaredInputs.getChannels()
-        if( inputs.size() == 1 )
-            return inputs.first().chainWith( it -> [it] )
-
-        final count = (0.. CH.isChannelQueue(inputs[i]) && !declaredInputs[i].isIterator()
-        )
-        if( NF.isStrictMode() && count > 1 )
-            throw new ScriptRuntimeException("Process `$name` received multiple queue channel inputs which will be implicitly mergeed -- consider combining them explicitly with `combine` or `join`, or converting single-item chennels into value channels with `collect` or `first`")
-
-        final iterators = (0.. declaredInputs[i].isIterator() )
-        return CH.getReadChannel(new CombineManyOp(inputs, iterators).apply())
+        final count = declaredInputs.count( param -> CH.isChannelQueue(param) && !param.isIterator() )
+        if( count > 1 ) {
+            final msg = "Process `$processName` received multiple queue channel inputs which will be implicitly mergeed -- consider combining them explicitly with `combine` or `join`, or converting single-item chennels into value channels with `collect` or `first`"
+            if( NF.isStrictMode() )
+                throw new ScriptRuntimeException(msg)
+            log.warn(msg)
+        }
+
+        final iterators = (0.. declaredInputs[i].isIterator() )
+        return CH.getReadChannel(new CombineManyOp(declaredInputs.getChannels(), iterators).apply())
     }
 
     private void collectOutputs(boolean singleton) {
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessInput.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessInput.groovy
index 5eccb7a294..89d417f459 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessInput.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessInput.groovy
@@ -34,8 +34,6 @@ class ProcessInput implements Cloneable {
 
     private String name
 
-    private Object arg
-
     private DataflowReadChannel channel
 
     /**
@@ -51,19 +49,11 @@ class ProcessInput implements Cloneable {
         return name
     }
 
-    void bind(Object arg) {
-        this.arg = arg
-        this.channel = getInChannel(arg)
+    void bind(Object value) {
+        this.channel = getInChannel(value)
     }
 
-    private DataflowReadChannel getInChannel(Object obj) {
-        if( obj == null )
-            throw new IllegalArgumentException('A process input channel evaluates to null')
-
-        def value = obj instanceof Closure
-            ? obj.call()
-            : obj
-
+    private DataflowReadChannel getInChannel(Object value) {
         if( value == null )
             throw new IllegalArgumentException('A process input channel evaluates to null')
 
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ScriptTokens.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ScriptTokens.groovy
index 7f719ad1fd..2d3d2339db 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ScriptTokens.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ScriptTokens.groovy
@@ -48,7 +48,7 @@ class TokenPathCall {
 
     TokenPathCall(target) {
         this.target = target
-        this.opts = Collections.emptyMap()
+        this.opts = [:]
     }
 
     TokenPathCall(Map opts, target) {
diff --git a/tests/singleton.nf b/tests/singleton.nf
index 2964fa27e1..7a0ed56f79 100644
--- a/tests/singleton.nf
+++ b/tests/singleton.nf
@@ -17,7 +17,7 @@
 
 process foo {
   output:
-  file x
+  file 'x'
 
   '''
   echo -n Hello > x

From 36510f461df3f16f687be0411ce13ea3f4bea7c2 Mon Sep 17 00:00:00 2001
From: Ben Sherman 
Date: Mon, 18 Dec 2023 12:57:43 -0600
Subject: [PATCH 27/36] Fix failing integration tests, minor changes

Signed-off-by: Ben Sherman 
---
 .../nextflow/extension/CombineManyOp.groovy   | 32 ++++++++++---------
 .../nextflow/script/ProcessFileInput.groovy   |  3 +-
 .../nextflow/script/dsl/ProcessDsl.groovy     | 18 +++++++----
 tests/blast-parallel-dsl2.nf                  |  4 +--
 tests/collect_and_merge.nf                    |  4 +--
 5 files changed, 33 insertions(+), 28 deletions(-)

diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/CombineManyOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/CombineManyOp.groovy
index 324f9e73ed..65ff4b0057 100644
--- a/modules/nextflow/src/main/groovy/nextflow/extension/CombineManyOp.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/extension/CombineManyOp.groovy
@@ -20,7 +20,9 @@ import java.util.concurrent.atomic.AtomicInteger
 
 import groovy.transform.CompileStatic
 import groovy.util.logging.Slf4j
+import groovyx.gpars.dataflow.DataflowQueue
 import groovyx.gpars.dataflow.DataflowReadChannel
+import groovyx.gpars.dataflow.DataflowVariable
 import groovyx.gpars.dataflow.DataflowWriteChannel
 import nextflow.Channel
 /**
@@ -55,14 +57,23 @@ class CombineManyOp {
         this.queues = sources.collect( ch -> [] )
         this.singletons = sources.collect( ch -> !CH.isChannelQueue(ch) )
         this.emitSingleton = iterators.size() == 0 && singletons.every()
-        this.emitCombination = iterators.size() == sources.size()
+        this.emitCombination = iterators.size() > 0 && singletons.every()
     }
 
-    private Map handler(int index, DataflowWriteChannel target, AtomicInteger counter) {
+    DataflowWriteChannel apply() {
+        final target = emitSingleton
+            ? new DataflowVariable()
+            : new DataflowQueue()
+        final counter = new AtomicInteger(sources.size())
+        for( int i = 0; i < sources.size(); i++ )
+            DataflowHelper.subscribeImpl( sources[i], eventsMap(i, target, counter) )
+
+        return target
+    }
+
+    private Map eventsMap(int index, DataflowWriteChannel target, AtomicInteger counter) {
         final opts = new LinkedHashMap(2)
-        opts.onNext = {
-            onNext(target, index, it)
-        }
+        opts.onNext = this.&take.curry(target, index)
         opts.onComplete = {
             if( counter.decrementAndGet() == 0 && !emitSingleton && !emitCombination )
                 target.bind(Channel.STOP)
@@ -70,7 +81,7 @@ class CombineManyOp {
         return opts
     }
 
-    private synchronized void onNext(DataflowWriteChannel target, int index, Object value) {
+    private synchronized void take(DataflowWriteChannel target, int index, Object value) {
         queues[index].add(value)
 
         // wait until every source has a value
@@ -122,13 +133,4 @@ class CombineManyOp {
             target.bind(new ArrayList(args))
         }
     }
-
-    DataflowWriteChannel apply() {
-        final target = CH.create(emitSingleton)
-        final counter = new AtomicInteger(sources.size())
-        for( int i = 0; i < sources.size(); i++ )
-            DataflowHelper.subscribeImpl( sources[i], handler(i, target, counter) )
-
-        return target
-    }
 }
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessFileInput.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessFileInput.groovy
index d468a96ab2..ce26d0d87f 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessFileInput.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessFileInput.groovy
@@ -43,13 +43,12 @@ class ProcessFileInput implements PathArityAware {
         this.value = value
         this.name = name
         this.pathQualifier = pathQualifier
-        this.filePattern = opts.stageAs ?: opts.name
 
         for( Map.Entry entry : opts )
             setProperty(entry.key, entry.value)
     }
 
-    void setStageAs(String value) {
+    void setStageAs(CharSequence value) {
         this.filePattern = value
     }
 
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessDsl.groovy b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessDsl.groovy
index 96ed502a3e..89ca0ca802 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessDsl.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessDsl.groovy
@@ -72,7 +72,6 @@ class ProcessDsl extends ProcessBuilder {
 
     void _in_env(LazyVar var) {
         final param = "\$in${inputs.size()}".toString()
-
         inputs.addParam(param)
         inputs.addEnv(var.name, new LazyVar(param))
     }
@@ -88,6 +87,9 @@ class ProcessDsl extends ProcessBuilder {
     }
 
     private String _in_path0(Object source, boolean pathQualifier, Map opts) {
+        if( !opts.stageAs && opts.name )
+            opts.stageAs = opts.remove('name')
+
         if( source instanceof LazyVar ) {
             final var = (LazyVar)source
             inputs.addFile(new ProcessFileInput(var, var.name, pathQualifier, opts))
@@ -96,7 +98,7 @@ class ProcessDsl extends ProcessBuilder {
         else if( source instanceof CharSequence ) {
             final param = "\$in${inputs.size()}"
             if( !opts.stageAs )
-                opts.stageAs = source.toString()
+                opts.stageAs = source
             inputs.addFile(new ProcessFileInput(new LazyVar(param), null, pathQualifier, opts))
             return param
         }
@@ -104,15 +106,17 @@ class ProcessDsl extends ProcessBuilder {
             throw new IllegalArgumentException()
     }
 
-    void _in_stdin(LazyVar var=null) {
-        final param = var != null
-            ? var.name
-            : "\$in${inputs.size()}".toString()
-
+    void _in_stdin() {
+        final param = "\$in${inputs.size()}".toString()
         inputs.addParam(param)
         inputs.stdin = new LazyVar(param)
     }
 
+    void _in_stdin(LazyVar var) {
+        inputs.addParam(var.name)
+        inputs.stdin = var
+    }
+
     @CompileDynamic
     void _in_tuple(Object... elements) {
         if( elements.length < 2 )
diff --git a/tests/blast-parallel-dsl2.nf b/tests/blast-parallel-dsl2.nf
index a2b3addbd7..9220684d2a 100644
--- a/tests/blast-parallel-dsl2.nf
+++ b/tests/blast-parallel-dsl2.nf
@@ -14,7 +14,7 @@ process blast {
     path 'query.fa'
 
     output:
-    path top_hits
+    path 'top_hits'
 
     """
     blastp -db ${db} -query query.fa -outfmt 6 > blast_result
@@ -27,7 +27,7 @@ process blast {
  */ 
 process extract {
     input:
-    path top_hits
+    path 'top_hits'
 
     output:
     path 'sequences'
diff --git a/tests/collect_and_merge.nf b/tests/collect_and_merge.nf
index 4a3bc773b4..0570a0eee3 100644
--- a/tests/collect_and_merge.nf
+++ b/tests/collect_and_merge.nf
@@ -27,7 +27,7 @@ process algn {
   each seq_id
 
   output:
-  tuple val(barcode), val(seq_id), file('bam'), file('bai')
+  tuple val(barcode), val(seq_id), path('bam'), path('bai')
 
   """
   echo BAM $seq_id - $barcode > bam
@@ -44,7 +44,7 @@ process merge {
   debug true
 
   input:
-  tuple val(barcode), val(seq_id), file(bam: 'bam?'), file(bai: 'bai?')
+  tuple val(barcode), val(seq_id), path(bam, stageAs: 'bam?'), path(bai, stageAs: 'bai?')
 
   """
   echo barcode: $barcode

From 353493ef5356d0776434f33d0fb053c98b4e205d Mon Sep 17 00:00:00 2001
From: Ben Sherman 
Date: Mon, 18 Dec 2023 13:12:04 -0600
Subject: [PATCH 28/36] Update tests

Signed-off-by: Ben Sherman 
---
 tests/singleton.nf          | 4 ++--
 tests/workdir-with-blank.nf | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/singleton.nf b/tests/singleton.nf
index 7a0ed56f79..cc39378928 100644
--- a/tests/singleton.nf
+++ b/tests/singleton.nf
@@ -17,7 +17,7 @@
 
 process foo {
   output:
-  file 'x'
+  path 'x'
 
   '''
   echo -n Hello > x
@@ -26,7 +26,7 @@ process foo {
 
 process bar {
   input:
-  file x
+  path x
   val y
 
   """
diff --git a/tests/workdir-with-blank.nf b/tests/workdir-with-blank.nf
index ade987ad20..6672615819 100644
--- a/tests/workdir-with-blank.nf
+++ b/tests/workdir-with-blank.nf
@@ -20,7 +20,7 @@ process foo {
     each x
 
     output:
-    file result_data
+    path 'result_data'
 
     """
     echo Hello $x > result_data

From 177120cb2c3a013547f91c34dda6665e7841833e Mon Sep 17 00:00:00 2001
From: Ben Sherman 
Date: Mon, 18 Dec 2023 14:13:38 -0600
Subject: [PATCH 29/36] Add comments

Signed-off-by: Ben Sherman 
---
 .../nextflow/extension/CombineManyOp.groovy   | 18 +++++++++++--
 .../nextflow/script/ProcessFileInput.groovy   | 13 ++++++++++
 .../nextflow/script/ProcessFileOutput.groovy  |  5 ++++
 .../nextflow/script/ProcessInput.groovy       |  8 ++++++
 .../nextflow/script/ProcessInputs.groovy      | 19 ++++++++++++++
 .../nextflow/script/ProcessOutput.groovy      | 25 +++++++++++++++++++
 .../nextflow/script/ProcessOutputs.groovy     | 10 ++++++++
 .../groovy/nextflow/util/LazyHelper.groovy    | 12 ++++-----
 8 files changed, 102 insertions(+), 8 deletions(-)

diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/CombineManyOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/CombineManyOp.groovy
index 65ff4b0057..3e0c662133 100644
--- a/modules/nextflow/src/main/groovy/nextflow/extension/CombineManyOp.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/extension/CombineManyOp.groovy
@@ -26,8 +26,8 @@ import groovyx.gpars.dataflow.DataflowVariable
 import groovyx.gpars.dataflow.DataflowWriteChannel
 import nextflow.Channel
 /**
- * Operator for combining many source channels into a single channel,
- * with the option to only merge channels that are not marked as "iterators".
+ * Operator for merging many source channels into a single channel,
+ * with the option to combine channels that are marked as "iterators".
  *
  * @see ProcessDef#collectInputs(Object[])
  *
@@ -41,12 +41,26 @@ class CombineManyOp {
 
     private List iterators
 
+    /**
+     * List of queues to receive values from source channels.
+     */
     private List queues = []
 
+    /**
+     * Mask of source channels that are singletons.
+     */
     private List singletons
 
+    /**
+     * True when all source channels are singletons and therefore
+     * the operator should emit a singleton channel.
+     */
     private boolean emitSingleton
 
+    /**
+     * True when all source channels are iterators and therefore
+     * the operator should simply emit the combinations.
+     */
     private boolean emitCombination
 
     private transient List combinations
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessFileInput.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessFileInput.groovy
index ce26d0d87f..f81cd65b5f 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessFileInput.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessFileInput.groovy
@@ -28,8 +28,17 @@ import nextflow.util.LazyHelper
 @CompileStatic
 class ProcessFileInput implements PathArityAware {
 
+    /**
+     * Lazy expression (e.g. lazy var, closure, GString) which
+     * defines which files to stage in terms of the task inputs.
+     * It is evaluated for each task against the task context.
+     */
     private Object value
 
+    /**
+     * Optional name which, if specified, will be added to the task
+     * context as an escape-aware list of paths.
+     */
     private String name
 
     /**
@@ -37,6 +46,10 @@ class ProcessFileInput implements PathArityAware {
      */
     private boolean pathQualifier
 
+    /**
+     * File pattern which defines how the input files should be named
+     * when they are staged into a task directory.
+     */
     private Object filePattern
 
     ProcessFileInput(Object value, String name, boolean pathQualifier, Map opts) {
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessFileOutput.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessFileOutput.groovy
index 9c5e43ccaa..12413d5c57 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessFileOutput.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessFileOutput.groovy
@@ -35,6 +35,11 @@ import nextflow.util.LazyHelper
 @CompileStatic
 class ProcessFileOutput implements PathArityAware {
 
+    /**
+     * Lazy expression (e.g. lazy var, closure, GString) which
+     * defines which files to unstage from the task directory.
+     * It will be evaluated for each task against the task directory.
+     */
     private Object target
 
     /**
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessInput.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessInput.groovy
index 89d417f459..3bfc428e86 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessInput.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessInput.groovy
@@ -32,8 +32,16 @@ import nextflow.extension.ToListOp
 @CompileStatic
 class ProcessInput implements Cloneable {
 
+    /**
+     * Parameter name under which the input value for each task
+     * will be added to the task context.
+     */
     private String name
 
+    /**
+     * Input channel which is created when the process is invoked
+     * in a workflow.
+     */
     private DataflowReadChannel channel
 
     /**
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessInputs.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessInputs.groovy
index d7ac0dd105..88912a7f11 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessInputs.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessInputs.groovy
@@ -28,12 +28,31 @@ class ProcessInputs implements List, Cloneable {
     @Delegate
     private List params = []
 
+    /**
+     * Input variables which will be evaluated for each task
+     * in terms of the task inputs and added to the task context.
+     */
     private Map vars = [:]
 
+    /**
+     * Environment variables which will be evaluated for each
+     * task against the task context and added to the task
+     * environment.
+     */
     private Map env = [:]
 
+    /**
+     * Input files which will be evaluated for each task
+     * against the task context and staged into the task
+     * directory.
+     */
     private List files = []
 
+    /**
+     * Lazy expression which will be evaluated for each task
+     * against the task context and provided as the standard
+     * input to the task.
+     */
     Object stdin
 
     @Override
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessOutput.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessOutput.groovy
index cf04c11a86..1d5006693c 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessOutput.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessOutput.groovy
@@ -32,16 +32,41 @@ import nextflow.util.LazyHelper
 @CompileStatic
 class ProcessOutput implements Cloneable {
 
+    /**
+     * List of declared outputs of the parent process.
+     */
     private ProcessOutputs declaredOutputs
 
+    /**
+     * Lazy expression (e.g. lazy var, closure, GString) which
+     * defines the output value in terms of the task context,
+     * including environment variables, files, and standard output.
+     * It will be evaluated for each task after it is executed. 
+     */
     private Object target
 
+    /**
+     * Optional parameter name under which the output channel
+     * is made available in the process outputs (i.e. `.out`).
+     */
     private String name
 
+    /**
+     * Optional channel topic which this output channel will
+     * be sent to.
+     */
     private String topic
 
+    /**
+     * When true, a task will not fail if any environment
+     * vars or files for this output are missing.
+     */
     private boolean optional
 
+    /**
+     * Output channel which is created when the process is invoked
+     * in a workflow.
+     */
     private DataflowWriteChannel channel
 
     ProcessOutput(ProcessOutputs declaredOutputs, Object target, Map opts) {
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessOutputs.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessOutputs.groovy
index 6e6c23ec7f..c7dfeb7a04 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessOutputs.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessOutputs.groovy
@@ -30,8 +30,18 @@ class ProcessOutputs implements List, Cloneable {
     @Delegate
     private List params = []
 
+    /**
+     * Environment variables which will be exported from the
+     * task environment for each task and made available to
+     * process outputs.
+     */
     private Map env = [:]
 
+    /**
+     * Output files which will be unstaged from the task
+     * directory for each task and made available to process
+     * outputs.
+     */
     private Map files = [:]
 
     @Override
diff --git a/modules/nextflow/src/main/groovy/nextflow/util/LazyHelper.groovy b/modules/nextflow/src/main/groovy/nextflow/util/LazyHelper.groovy
index 6c6e0ad7a8..da6c368c35 100644
--- a/modules/nextflow/src/main/groovy/nextflow/util/LazyHelper.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/util/LazyHelper.groovy
@@ -20,7 +20,7 @@ import groovy.transform.CompileStatic
 import groovy.transform.EqualsAndHashCode
 import groovy.transform.ToString
 /**
- * Helper methods for lazy binding and resolution.
+ * Helper methods for lazy evaluation.
  *
  * @author Paolo Di Tommaso 
  * @author Ben Sherman 
@@ -29,7 +29,7 @@ import groovy.transform.ToString
 class LazyHelper {
 
     /**
-     * Resolve a lazy value against a given binding.
+     * Evaluate a lazy expression against a given binding.
      *
      * @param binding
      * @param value
@@ -50,14 +50,14 @@ class LazyHelper {
 }
 
 /**
- * Interface for types that can be lazily resolved
+ * Interface for types that can be lazily evaluated
  */
 interface LazyAware {
     Object resolve(Object binding)
 }
 
 /**
- * A list that can be lazily resolved
+ * A list that can be lazily evaluated
  */
 @CompileStatic
 class LazyList implements LazyAware, List {
@@ -88,7 +88,7 @@ class LazyList implements LazyAware, List {
 }
 
 /**
- * A map whose values can be lazily resolved
+ * A map whose values can be lazily evaluated
  */
 @CompileStatic
 class LazyMap implements Map {
@@ -263,7 +263,7 @@ class LazyMap implements Map {
 }
 
 /**
- * A variable that can be lazily resolved
+ * A variable that can be lazily evaluated
  */
 @CompileStatic
 @EqualsAndHashCode

From 8441989c41f34618edf614b4a5880ab875d135b1 Mon Sep 17 00:00:00 2001
From: Ben Sherman 
Date: Mon, 18 Dec 2023 14:13:50 -0600
Subject: [PATCH 30/36] Fix stdout evaluation

Signed-off-by: Ben Sherman 
---
 .../nextflow/processor/TaskOutputCollector.groovy      | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskOutputCollector.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskOutputCollector.groovy
index 9ca8c0747d..865b04a94d 100644
--- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskOutputCollector.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskOutputCollector.groovy
@@ -91,15 +91,15 @@ class TaskOutputCollector implements Map {
      * Get the standard output from the task environment.
      */
     Object stdout() {
-        final result = task.getStdout()
+        final value = task.@stdout
 
-        if( result == null && task.type == ScriptType.SCRIPTLET )
+        if( value == null && task.type == ScriptType.SCRIPTLET )
             throw new IllegalArgumentException("Missing 'stdout' for process > ${task.lazyName()}")
 
-        if( result instanceof Path && !result.exists() )
-            throw new MissingFileException("Missing 'stdout' file: ${result.toUriString()} for process > ${task.lazyName()}")
+        if( value instanceof Path && !value.exists() )
+            throw new MissingFileException("Missing 'stdout' file: ${value.toUriString()} for process > ${task.lazyName()}")
 
-        return result
+        return value instanceof Path ? ((Path)value).text : value?.toString()
     }
 
     /**

From 1f6705bdec685c1154c8afb44ecf218c46294de1 Mon Sep 17 00:00:00 2001
From: Ben Sherman 
Date: Thu, 28 Mar 2024 01:05:20 -0500
Subject: [PATCH 31/36] Move LazyHelper to script package, update copyright

Signed-off-by: Ben Sherman 
---
 .../src/main/groovy/nextflow/ast/NextflowDSLImpl.groovy   | 2 +-
 .../groovy/nextflow/exception/ProcessEvalException.groovy | 2 +-
 .../main/groovy/nextflow/extension/CombineManyOp.groovy   | 2 +-
 .../src/main/groovy/nextflow/processor/TaskConfig.groovy  | 2 +-
 .../groovy/nextflow/processor/TaskEnvCollector.groovy     | 2 +-
 .../groovy/nextflow/processor/TaskFileCollector.groovy    | 2 +-
 .../groovy/nextflow/processor/TaskOutputCollector.groovy  | 2 +-
 .../main/groovy/nextflow/processor/TaskProcessor.groovy   | 2 +-
 .../src/main/groovy/nextflow/processor/TaskRun.groovy     | 1 +
 .../src/main/groovy/nextflow/script/IncludeDef.groovy     | 1 -
 .../groovy/nextflow/{util => script}/LazyHelper.groovy    | 4 ++--
 .../main/groovy/nextflow/script/ProcessFileInput.groovy   | 3 +--
 .../main/groovy/nextflow/script/ProcessFileOutput.groovy  | 3 +--
 .../src/main/groovy/nextflow/script/ProcessInput.groovy   | 2 +-
 .../src/main/groovy/nextflow/script/ProcessInputs.groovy  | 2 +-
 .../src/main/groovy/nextflow/script/ProcessOutput.groovy  | 3 +--
 .../src/main/groovy/nextflow/script/ProcessOutputs.groovy | 3 +--
 .../main/groovy/nextflow/script/dsl/ProcessBuilder.groovy | 4 ++--
 .../nextflow/script/dsl/ProcessConfigBuilder.groovy       | 2 +-
 .../src/main/groovy/nextflow/script/dsl/ProcessDsl.groovy | 8 ++++----
 .../groovy/nextflow/script/dsl/WorkflowBuilder.groovy     | 2 +-
 .../src/test/groovy/nextflow/processor/TaskRunTest.groovy | 2 +-
 .../groovy/nextflow/script/dsl/ProcessBuilderTest.groovy  | 4 ++--
 .../groovy/nextflow/script/params/CmdEvalParamTest.groovy | 2 +-
 .../groovy/nextflow/script/params/ParamsOutTest.groovy    | 4 ++--
 25 files changed, 31 insertions(+), 35 deletions(-)
 rename modules/nextflow/src/main/groovy/nextflow/{util => script}/LazyHelper.groovy (99%)

diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSLImpl.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSLImpl.groovy
index efc8c5aba3..be6cfc7f37 100644
--- a/modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSLImpl.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSLImpl.groovy
@@ -26,6 +26,7 @@ import nextflow.NF
 import nextflow.script.BaseScript
 import nextflow.script.BodyDef
 import nextflow.script.IncludeDef
+import nextflow.script.LazyVar
 import nextflow.script.TaskClosure
 import nextflow.script.TokenEvalCall
 import nextflow.script.TokenEnvCall
@@ -35,7 +36,6 @@ import nextflow.script.TokenStdinCall
 import nextflow.script.TokenStdoutCall
 import nextflow.script.TokenValCall
 import nextflow.script.TokenValRef
-import nextflow.util.LazyVar
 import org.codehaus.groovy.ast.ASTNode
 import org.codehaus.groovy.ast.ClassCodeVisitorSupport
 import org.codehaus.groovy.ast.ClassNode
diff --git a/modules/nextflow/src/main/groovy/nextflow/exception/ProcessEvalException.groovy b/modules/nextflow/src/main/groovy/nextflow/exception/ProcessEvalException.groovy
index a2babd2b2e..995112411a 100644
--- a/modules/nextflow/src/main/groovy/nextflow/exception/ProcessEvalException.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/exception/ProcessEvalException.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/CombineManyOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/CombineManyOp.groovy
index 3e0c662133..a76d47e7ea 100644
--- a/modules/nextflow/src/main/groovy/nextflow/extension/CombineManyOp.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/extension/CombineManyOp.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskConfig.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskConfig.groovy
index f95843727f..63e8db1057 100644
--- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskConfig.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskConfig.groovy
@@ -30,11 +30,11 @@ import nextflow.executor.BashWrapperBuilder
 import nextflow.executor.res.AcceleratorResource
 import nextflow.executor.res.DiskResource
 import nextflow.k8s.model.PodOptions
+import nextflow.script.LazyMap
 import nextflow.script.TaskClosure
 import nextflow.util.CmdLineHelper
 import nextflow.util.CmdLineOptionMap
 import nextflow.util.Duration
-import nextflow.util.LazyMap
 import nextflow.util.MemoryUnit
 /**
  * Task local configuration properties
diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskEnvCollector.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskEnvCollector.groovy
index 1e387a86b7..4e21763f66 100644
--- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskEnvCollector.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskEnvCollector.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskFileCollector.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskFileCollector.groovy
index 2f977f3b70..b5b00cab01 100644
--- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskFileCollector.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskFileCollector.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskOutputCollector.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskOutputCollector.groovy
index 9464538798..e19810e4a6 100644
--- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskOutputCollector.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskOutputCollector.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy
index 0de76f3d3e..a528a10e97 100644
--- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy
@@ -77,6 +77,7 @@ import nextflow.file.FileHolder
 import nextflow.file.FilePorter
 import nextflow.script.BaseScript
 import nextflow.script.BodyDef
+import nextflow.script.LazyHelper
 import nextflow.script.ProcessConfig
 import nextflow.script.ScriptMeta
 import nextflow.script.ScriptType
@@ -86,7 +87,6 @@ import nextflow.util.ArrayBag
 import nextflow.util.BlankSeparatedList
 import nextflow.util.CacheHelper
 import nextflow.util.Escape
-import nextflow.util.LazyHelper
 import nextflow.util.LockManager
 import nextflow.util.LoggerHelper
 import nextflow.util.TestOnly
diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy
index a4011eed6a..1f021382b9 100644
--- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy
@@ -36,6 +36,7 @@ import nextflow.exception.ProcessUnrecoverableException
 import nextflow.file.FileHelper
 import nextflow.file.FileHolder
 import nextflow.script.BodyDef
+import nextflow.script.LazyHelper
 import nextflow.script.ScriptType
 import nextflow.script.TaskClosure
 import nextflow.script.bundle.ResourcesBundle
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/IncludeDef.groovy b/modules/nextflow/src/main/groovy/nextflow/script/IncludeDef.groovy
index 4bff5edf22..6b91d6d716 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/IncludeDef.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/IncludeDef.groovy
@@ -32,7 +32,6 @@ import groovy.util.logging.Slf4j
 import nextflow.NF
 import nextflow.Session
 import nextflow.exception.IllegalModulePath
-import nextflow.util.LazyVar
 /**
  * Implements a script inclusion
  *
diff --git a/modules/nextflow/src/main/groovy/nextflow/util/LazyHelper.groovy b/modules/nextflow/src/main/groovy/nextflow/script/LazyHelper.groovy
similarity index 99%
rename from modules/nextflow/src/main/groovy/nextflow/util/LazyHelper.groovy
rename to modules/nextflow/src/main/groovy/nextflow/script/LazyHelper.groovy
index da6c368c35..e28ea92118 100644
--- a/modules/nextflow/src/main/groovy/nextflow/util/LazyHelper.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/LazyHelper.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package nextflow.util
+package nextflow.script
 
 import groovy.transform.CompileStatic
 import groovy.transform.EqualsAndHashCode
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessFileInput.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessFileInput.groovy
index f81cd65b5f..bb4dfc93a9 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessFileInput.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessFileInput.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,7 +17,6 @@
 package nextflow.script
 
 import groovy.transform.CompileStatic
-import nextflow.util.LazyHelper
 
 /**
  * Models a process file input, which defines a file
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessFileOutput.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessFileOutput.groovy
index 12413d5c57..4d364cc8e2 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessFileOutput.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessFileOutput.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -23,7 +23,6 @@ import groovy.util.logging.Slf4j
 import nextflow.exception.IllegalFileException
 import nextflow.file.FilePatternSplitter
 import nextflow.util.BlankSeparatedList
-import nextflow.util.LazyHelper
 /**
  * Models a process file output, which defines a file
  * or set of files to be unstaged from a task work directory.
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessInput.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessInput.groovy
index b7b561aa5b..bae5d200a8 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessInput.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessInput.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessInputs.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessInputs.groovy
index 9886718e54..072e2d2509 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessInputs.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessInputs.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessOutput.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessOutput.groovy
index b85404146f..694eb71007 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessOutput.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessOutput.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -22,7 +22,6 @@ import groovyx.gpars.dataflow.DataflowWriteChannel
 import nextflow.processor.TaskOutputCollector
 import nextflow.processor.TaskRun
 import nextflow.util.ConfigHelper
-import nextflow.util.LazyHelper
 /**
  * Models a process output.
  *
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessOutputs.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessOutputs.groovy
index 29a319cf2f..b82fab34e8 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessOutputs.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessOutputs.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -18,7 +18,6 @@ package nextflow.script
 
 import groovyx.gpars.dataflow.DataflowQueue
 import groovyx.gpars.dataflow.DataflowWriteChannel
-import nextflow.util.LazyVar
 
 /**
  * Models the process outputs.
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessBuilder.groovy b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessBuilder.groovy
index a85ff72243..7fb9615d46 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessBuilder.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessBuilder.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -28,9 +28,9 @@ import nextflow.script.ProcessInputs
 import nextflow.script.ProcessOutputs
 import nextflow.script.BaseScript
 import nextflow.script.BodyDef
+import nextflow.script.LazyList
 import nextflow.script.ProcessConfig
 import nextflow.script.ProcessDef
-import nextflow.util.LazyList
 
 /**
  * Builder for {@link ProcessDef}.
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessConfigBuilder.groovy b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessConfigBuilder.groovy
index 4559e6492c..f522368287 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessConfigBuilder.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessConfigBuilder.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessDsl.groovy b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessDsl.groovy
index dc017b17a3..0cf0b575ab 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessDsl.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessDsl.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -20,6 +20,9 @@ import groovy.transform.CompileDynamic
 import groovy.transform.CompileStatic
 import nextflow.processor.TaskOutputCollector
 import nextflow.script.BaseScript
+import nextflow.script.LazyAware
+import nextflow.script.LazyList
+import nextflow.script.LazyVar
 import nextflow.script.ProcessDef
 import nextflow.script.ProcessFileInput
 import nextflow.script.ProcessFileOutput
@@ -33,9 +36,6 @@ import nextflow.script.TokenPathCall
 import nextflow.script.TokenStdinCall
 import nextflow.script.TokenStdoutCall
 import nextflow.script.TokenValCall
-import nextflow.util.LazyAware
-import nextflow.util.LazyList
-import nextflow.util.LazyVar
 
 /**
  * Implements the process DSL.
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/dsl/WorkflowBuilder.groovy b/modules/nextflow/src/main/groovy/nextflow/script/dsl/WorkflowBuilder.groovy
index 2e9e1497d5..cd79b3f3a7 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/dsl/WorkflowBuilder.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/dsl/WorkflowBuilder.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/test/groovy/nextflow/processor/TaskRunTest.groovy b/modules/nextflow/src/test/groovy/nextflow/processor/TaskRunTest.groovy
index 4882c2763c..e0b8a8b0e5 100644
--- a/modules/nextflow/src/test/groovy/nextflow/processor/TaskRunTest.groovy
+++ b/modules/nextflow/src/test/groovy/nextflow/processor/TaskRunTest.groovy
@@ -27,9 +27,9 @@ import nextflow.container.resolver.ContainerInfo
 import nextflow.executor.Executor
 import nextflow.file.FileHolder
 import nextflow.script.BodyDef
+import nextflow.script.LazyVar
 import nextflow.script.ScriptBinding
 import nextflow.script.TaskClosure
-import nextflow.util.LazyVar
 import nextflow.script.params.EnvInParam
 import nextflow.script.params.EnvOutParam
 import nextflow.script.params.FileInParam
diff --git a/modules/nextflow/src/test/groovy/nextflow/script/dsl/ProcessBuilderTest.groovy b/modules/nextflow/src/test/groovy/nextflow/script/dsl/ProcessBuilderTest.groovy
index b404796d4e..ff2682e7ee 100644
--- a/modules/nextflow/src/test/groovy/nextflow/script/dsl/ProcessBuilderTest.groovy
+++ b/modules/nextflow/src/test/groovy/nextflow/script/dsl/ProcessBuilderTest.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -25,8 +25,8 @@ import nextflow.script.params.StdInParam
 import nextflow.script.params.StdOutParam
 import nextflow.script.params.ValueInParam
 import nextflow.script.BaseScript
+import nextflow.script.LazyVar
 import nextflow.script.ProcessConfig
-import nextflow.util.LazyVar
 import nextflow.util.Duration
 import nextflow.util.MemoryUnit
 /**
diff --git a/modules/nextflow/src/test/groovy/nextflow/script/params/CmdEvalParamTest.groovy b/modules/nextflow/src/test/groovy/nextflow/script/params/CmdEvalParamTest.groovy
index 39cbc26322..b591c384fd 100644
--- a/modules/nextflow/src/test/groovy/nextflow/script/params/CmdEvalParamTest.groovy
+++ b/modules/nextflow/src/test/groovy/nextflow/script/params/CmdEvalParamTest.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/test/groovy/nextflow/script/params/ParamsOutTest.groovy b/modules/nextflow/src/test/groovy/nextflow/script/params/ParamsOutTest.groovy
index a0de17d176..b7d0a69d5b 100644
--- a/modules/nextflow/src/test/groovy/nextflow/script/params/ParamsOutTest.groovy
+++ b/modules/nextflow/src/test/groovy/nextflow/script/params/ParamsOutTest.groovy
@@ -21,9 +21,9 @@ import static test.TestParser.*
 import java.nio.file.Path
 
 import groovyx.gpars.dataflow.DataflowVariable
-import nextflow.script.PathArityAware
 import nextflow.processor.TaskContext
-import nextflow.util.LazyVar
+import nextflow.script.LazyVar
+import nextflow.script.PathArityAware
 import nextflow.util.BlankSeparatedList
 import test.Dsl2Spec
 /**

From ecdaaa4c1908bac1a13b96327790ab43ec6fc419 Mon Sep 17 00:00:00 2001
From: Ben Sherman 
Date: Thu, 28 Mar 2024 01:20:47 -0500
Subject: [PATCH 32/36] cleanup

Signed-off-by: Ben Sherman 
---
 .../processor/TaskOutputCollector.groovy        | 17 ++++++++---------
 .../groovy/nextflow/processor/TaskRun.groovy    |  2 +-
 .../nextflow/script/ProcessOutputs.groovy       | 12 +++++++-----
 .../nextflow/script/dsl/ProcessDsl.groovy       | 16 +++++++---------
 4 files changed, 23 insertions(+), 24 deletions(-)

diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskOutputCollector.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskOutputCollector.groovy
index e19810e4a6..ffeb83106f 100644
--- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskOutputCollector.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskOutputCollector.groovy
@@ -55,14 +55,13 @@ class TaskOutputCollector implements Map {
     /**
      * Get an environment variable from the task environment.
      *
-     * @param key
+     * @param name
      */
-    String env(String key) {
-        final varName = declaredOutputs.getEnv().get(key)
-        final result = env0(task.workDir).get(varName)
+    String env(String name) {
+        final result = env0(task.workDir).get(name)
 
         if( result == null && !optional )
-            throw new MissingValueException("Missing environment variable: $varName")
+            throw new MissingValueException("Missing environment variable: $name")
 
         return result
     }
@@ -70,14 +69,14 @@ class TaskOutputCollector implements Map {
     /**
      * Get the result of an eval command from the task environment.
      *
-     * @param varName
+     * @param name
      */
-    String eval(String varName) {
+    String eval(String name) {
         final evalCmds = task.getOutputEvals()
-        final result = env0(task.workDir, evalCmds).get(varName)
+        final result = env0(task.workDir, evalCmds).get(name)
 
         if( result == null && !optional )
-            throw new MissingValueException("Missing result of eval command: '${evalCmds.get(varName)}'")
+            throw new MissingValueException("Missing result of eval command: '${evalCmds.get(name)}'")
 
         return result
     }
diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy
index 1f021382b9..1753a10842 100644
--- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy
@@ -505,7 +505,7 @@ class TaskRun implements Cloneable {
 
     List getOutputEnvNames() {
         final declaredOutputs = processor.config.getOutputs()
-        return new ArrayList(declaredOutputs.env.values())
+        return new ArrayList(declaredOutputs.getEnv())
     }
 
     Map getOutputEvals() {
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessOutputs.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessOutputs.groovy
index b82fab34e8..0e993edc07 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessOutputs.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessOutputs.groovy
@@ -34,7 +34,7 @@ class ProcessOutputs implements List, Cloneable {
      * task environment for each task and made available to
      * process outputs.
      */
-    private Map env = [:]
+    private Set env = []
 
     /**
      * Shell commands which will be executed in the task environment
@@ -61,8 +61,8 @@ class ProcessOutputs implements List, Cloneable {
         params.add(param)
     }
 
-    void addEnv(String name, String value) {
-        env.put(name, value)
+    void addEnv(String name) {
+        env.add(name)
     }
 
     String addEval(Object value) {
@@ -71,8 +71,10 @@ class ProcessOutputs implements List, Cloneable {
         return key
     }
 
-    void addFile(String key, ProcessFileOutput file) {
+    String addFile(ProcessFileOutput file) {
+        final key = "\$file${files.size()}"
         files.put(key, file)
+        return key
     }
 
     List getNames() {
@@ -83,7 +85,7 @@ class ProcessOutputs implements List, Cloneable {
         return params*.getChannel()
     }
 
-    Map getEnv() {
+    Set getEnv() {
         return env
     }
 
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessDsl.groovy b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessDsl.groovy
index 0cf0b575ab..556e96ca7a 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessDsl.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessDsl.groovy
@@ -175,7 +175,7 @@ class ProcessDsl extends ProcessBuilder {
             opts.name = opts.remove('emit')
 
         final name = _out_env0(target)
-        outputs.addEnv(name, name)
+        outputs.addEnv(name)
         outputs.addParam(new LazyEnvCall(name), opts)
     }
 
@@ -226,9 +226,7 @@ class ProcessDsl extends ProcessBuilder {
     }
 
     private String _out_path0(Object target, boolean pathQualifier, Map opts) {
-        final key = "\$file${outputs.getFiles().size()}".toString()
-        outputs.addFile(key, new ProcessFileOutput(target, pathQualifier, opts))
-        return key
+        outputs.addFile(new ProcessFileOutput(target, pathQualifier, opts))
     }
 
     void _out_stdout(Map opts=[:]) {
@@ -261,7 +259,7 @@ class ProcessDsl extends ProcessBuilder {
             }
             else if( item instanceof TokenEnvCall ) {
                 final name = _out_env0(item.val)
-                outputs.addEnv(name, name)
+                outputs.addEnv(name)
                 target << new LazyEnvCall(name)
             }
             else if( item instanceof TokenEvalCall ) {
@@ -329,10 +327,10 @@ class LazyTupleElement extends LazyVar {
 
 @CompileStatic
 class LazyEnvCall implements LazyAware {
-    String key
+    String name
 
-    LazyEnvCall(String key) {
-        this.key = key
+    LazyEnvCall(String name) {
+        this.name = name
     }
 
     @Override
@@ -340,7 +338,7 @@ class LazyEnvCall implements LazyAware {
         if( binding !instanceof TaskOutputCollector )
             throw new IllegalStateException()
 
-        ((TaskOutputCollector)binding).env(key)
+        ((TaskOutputCollector)binding).env(name)
     }
 }
 

From 4509b285f46c4d4888aa72131dcfad41c7c857db Mon Sep 17 00:00:00 2001
From: Ben Sherman 
Date: Thu, 28 Mar 2024 21:08:24 -0500
Subject: [PATCH 33/36] Infer staging of file inputs from input types

Signed-off-by: Ben Sherman 
---
 .../nextflow/ast/NextflowDSLImpl.groovy       | 147 +++++++------
 .../nextflow/ast/ProcessInputPathXform.groovy |  29 +++
 .../ast/ProcessInputPathXformImpl.groovy      | 202 ++++++++++++++++++
 .../nextflow/ast/ProcessOutputVisitor.groovy  | 144 +++++++++++++
 .../nextflow/processor/TaskProcessor.groovy   |   7 +-
 .../nextflow/script/ProcessInput.groovy       |  12 +-
 .../nextflow/script/ProcessInputs.groovy      |   4 +-
 .../nextflow/script/ProcessOutput.groovy      |   6 +
 .../nextflow/script/ProcessOutputs.groovy     |  10 +-
 .../nextflow/script/ScriptParser.groovy       |   2 +
 .../nextflow/script/dsl/ProcessDsl.groovy     |  57 ++++-
 11 files changed, 542 insertions(+), 78 deletions(-)
 create mode 100644 modules/nextflow/src/main/groovy/nextflow/ast/ProcessInputPathXform.groovy
 create mode 100644 modules/nextflow/src/main/groovy/nextflow/ast/ProcessInputPathXformImpl.groovy
 create mode 100644 modules/nextflow/src/main/groovy/nextflow/ast/ProcessOutputVisitor.groovy

diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSLImpl.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSLImpl.groovy
index be6cfc7f37..328414d51e 100644
--- a/modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSLImpl.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSLImpl.groovy
@@ -47,6 +47,7 @@ import org.codehaus.groovy.ast.expr.BinaryExpression
 import org.codehaus.groovy.ast.expr.CastExpression
 import org.codehaus.groovy.ast.expr.ClosureExpression
 import org.codehaus.groovy.ast.expr.ConstantExpression
+import org.codehaus.groovy.ast.expr.DeclarationExpression
 import org.codehaus.groovy.ast.expr.Expression
 import org.codehaus.groovy.ast.expr.GStringExpression
 import org.codehaus.groovy.ast.expr.ListExpression
@@ -512,6 +513,7 @@ class NextflowDSLImpl implements ASTTransformation {
                  * - collect all the statement after the 'exec:' label
                  */
                 def source = new StringBuilder()
+                List paramStatements = []
                 List execStatements = []
 
                 List whenStatements = []
@@ -535,7 +537,10 @@ class NextflowDSLImpl implements ASTTransformation {
                             if( stm instanceof ExpressionStatement ) {
                                 fixLazyGString( stm )
                                 fixStdinStdout( stm )
-                                convertInputMethod( stm.getExpression() )
+                                if( stm.expression instanceof DeclarationExpression )
+                                    convertInputDeclaration( stm )
+                                else if( stm.expression instanceof MethodCallExpression )
+                                    convertInputMethod( (MethodCallExpression)stm.expression )
                             }
                             break
 
@@ -543,7 +548,10 @@ class NextflowDSLImpl implements ASTTransformation {
                             if( stm instanceof ExpressionStatement ) {
                                 fixLazyGString( stm )
                                 fixStdinStdout( stm )
-                                convertOutputMethod( stm.getExpression() )
+                                if( stm.expression instanceof DeclarationExpression )
+                                    paramStatements.addAll( convertOutputDeclaration( stm ) )
+                                else if( stm.expression instanceof MethodCallExpression )
+                                    convertOutputMethod( (MethodCallExpression)stm.expression )
                             }
                             break
 
@@ -669,6 +677,10 @@ class NextflowDSLImpl implements ASTTransformation {
                     stm.visit(new TaskCmdXformVisitor(unit))
                 }
 
+                // prepend additional param statements
+                paramStatements.addAll(block.statements)
+                block.statements = paramStatements
+
                 if (!done) {
                     log.trace "Invalid 'process' definition -- Process must terminate with string expression"
                     int line = methodCall.lineNumber
@@ -859,59 +871,44 @@ class NextflowDSLImpl implements ASTTransformation {
             }
         }
 
-        /*
-         * handle *input* parameters
-         */
-        protected void convertInputMethod( Expression expression ) {
-            log.trace "convert > input expression: $expression"
-
-            if( expression instanceof MethodCallExpression ) {
-
-                def methodCall = expression as MethodCallExpression
-                def methodName = methodCall.getMethodAsString()
-                def nested = methodCall.objectExpression instanceof MethodCallExpression
-                log.trace "convert > input method: $methodName"
+        protected void convertInputDeclaration( ExpressionStatement stmt ) {
+            // don't throw error if not method because it could be an implicit script statement
+            if( stmt.expression !instanceof DeclarationExpression )
+                return
 
-                if( methodName in ['val','env','file','each','set','stdin','path','tuple'] ) {
-                    //this methods require a special prefix
-                    if( !nested )
-                        methodCall.setMethod( new ConstantExpression('_in_' + methodName) )
+            final decl = (DeclarationExpression)stmt.expression
+            if( decl.isMultipleAssignmentDeclaration() ) {
+                syntaxError(decl, "Invalid process input statement, possible syntax error")
+                return
+            }
 
-                    fixMethodCall(methodCall)
-                }
+            // NOTE: doint this in semantic analysis causes null pointer exception
+            final var = decl.variableExpression
+            stmt.expression = callThisX(
+                '_typed_in_param',
+                args(constX(var.name), classX(var.type))
+            )
+        }
 
-                /*
-                 * Handles a GString a file name, like this:
-                 *
-                 *      input:
-                 *        file x name "$var_name" from q
-                 *
-                 */
-                else if( methodName == 'name' && isWithinMethod(expression, 'file') ) {
-                    varToConstX(methodCall.getArguments())
-                }
+        private static final VALID_INPUT_METHODS = ['val','env','file','path','stdin','each','tuple']
 
-                // invoke on the next method call
-                if( expression.objectExpression instanceof MethodCallExpression ) {
-                    convertInputMethod(methodCall.objectExpression)
-                }
-            }
+        protected void convertInputMethod( MethodCallExpression methodCall ) {
+            final methodName = methodCall.getMethodAsString()
+            log.trace "convert > input method: $methodName"
 
-            else if( expression instanceof PropertyExpression ) {
-                // invoke on the next method call
-                if( expression.objectExpression instanceof MethodCallExpression ) {
-                    convertInputMethod(expression.objectExpression)
-                }
+            final caller = methodCall.objectExpression
+            if( caller !instanceof VariableExpression || caller.getText() != 'this' ) {
+                syntaxError(methodCall, "Invalid process input statement, possible syntax error")
+                return
             }
 
-        }
-
-        protected boolean isWithinMethod(MethodCallExpression method, String name) {
-            if( method.objectExpression instanceof MethodCallExpression ) {
-                return isWithinMethod(method.objectExpression as MethodCallExpression, name)
+            if( methodName !in VALID_INPUT_METHODS ) {
+                syntaxError(methodCall, "Invalid process input method '${methodName}'")
+                return
             }
 
-            return method.getMethodAsString() == name
+            methodCall.setMethod( new ConstantExpression('_in_' + methodName) )
+            fixMethodCall(methodCall)
         }
 
         /**
@@ -944,34 +941,56 @@ class NextflowDSLImpl implements ASTTransformation {
             }
         }
 
-        protected void convertOutputMethod( Expression expression ) {
-            log.trace "convert > output expression: $expression"
+        protected List convertOutputDeclaration( ExpressionStatement stmt ) {
+            // don't throw error if not method because it could be an implicit script statement
+            if( stmt.expression !instanceof DeclarationExpression )
+                return
+
+            final decl = (DeclarationExpression)stmt.expression
+            log.trace "convert > output declaration: $decl"
 
-            if( !(expression instanceof MethodCallExpression) ) {
+            if( decl.isMultipleAssignmentDeclaration() ) {
+                syntaxError(decl, "Invalid process output statement, possible syntax error")
                 return
             }
 
-            def methodCall = expression as MethodCallExpression
-            def methodName = methodCall.getMethodAsString()
-            def nested = methodCall.objectExpression instanceof MethodCallExpression
-            log.trace "convert > output method: $methodName"
+            final var = decl.variableExpression
+            final rhs = decl.rightExpression ?: var
+            stmt.expression = callThisX(
+                '_typed_out_param',
+                new ArgumentListExpression(
+                    constX(var.name),
+                    classX(var.type),
+                    closureX(new ExpressionStatement(rhs))
+                )
+            )
+
+            // infer unstaging directives from AST
+            final visitor = new ProcessOutputVisitor(unit)
+            rhs.visit(visitor)
+            return visitor.statements
+        }
 
-            if( methodName in ['val','env','eval','file','set','stdout','path','tuple'] && !nested ) {
-                // prefix the method name with the string '_out_'
-                methodCall.setMethod( new ConstantExpression('_out_' + methodName) )
-                fixMethodCall(methodCall)
-                fixOutEmitAndTopicOptions(methodCall)
-            }
+        private static final VALID_OUTPUT_METHODS = ['val','env','eval','file','path','stdout','tuple']
+
+        protected void convertOutputMethod( MethodCallExpression methodCall ) {
+            final methodName = methodCall.getMethodAsString()
+            log.trace "convert > output method: $methodName"
 
-            else if( methodName in ['into','mode'] ) {
-                fixMethodCall(methodCall)
+            final caller = methodCall.objectExpression
+            if( caller !instanceof VariableExpression || caller.getText() != 'this' ) {
+                syntaxError(methodCall, "Invalid process output statement, possible syntax error")
+                return
             }
 
-            // continue to traverse
-            if( methodCall.objectExpression instanceof MethodCallExpression ) {
-                convertOutputMethod(methodCall.objectExpression)
+            if( methodName !in VALID_OUTPUT_METHODS ) {
+                syntaxError(methodCall, "Invalid process output method '${methodName}'")
+                return
             }
 
+            methodCall.setMethod( new ConstantExpression('_out_' + methodName) )
+            fixMethodCall(methodCall)
+            fixOutEmitAndTopicOptions(methodCall)
         }
 
         private boolean withinTupleMethod
diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/ProcessInputPathXform.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/ProcessInputPathXform.groovy
new file mode 100644
index 0000000000..d7a5a0f401
--- /dev/null
+++ b/modules/nextflow/src/main/groovy/nextflow/ast/ProcessInputPathXform.groovy
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2013-2024, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package nextflow.ast
+
+import java.lang.annotation.ElementType
+import java.lang.annotation.Retention
+import java.lang.annotation.RetentionPolicy
+import java.lang.annotation.Target
+
+import org.codehaus.groovy.transform.GroovyASTTransformationClass
+
+@Retention(RetentionPolicy.SOURCE)
+@Target(ElementType.METHOD)
+@GroovyASTTransformationClass(classes = [ProcessInputPathXformImpl])
+@interface ProcessInputPathXform {}
diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/ProcessInputPathXformImpl.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/ProcessInputPathXformImpl.groovy
new file mode 100644
index 0000000000..b1daa98997
--- /dev/null
+++ b/modules/nextflow/src/main/groovy/nextflow/ast/ProcessInputPathXformImpl.groovy
@@ -0,0 +1,202 @@
+/*
+ * Copyright 2013-2024, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package nextflow.ast
+
+import java.lang.reflect.Field
+import java.lang.reflect.Modifier
+import java.lang.reflect.ParameterizedType
+import java.nio.file.Path
+
+import static org.codehaus.groovy.ast.tools.GeneralUtils.*
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.codehaus.groovy.ast.ASTNode
+import org.codehaus.groovy.ast.ClassCodeVisitorSupport
+import org.codehaus.groovy.ast.ClassNode
+import org.codehaus.groovy.ast.Parameter
+import org.codehaus.groovy.ast.VariableScope
+import org.codehaus.groovy.ast.expr.ArgumentListExpression
+import org.codehaus.groovy.ast.expr.ClassExpression
+import org.codehaus.groovy.ast.expr.ClosureExpression
+import org.codehaus.groovy.ast.expr.ConstantExpression
+import org.codehaus.groovy.ast.expr.Expression
+import org.codehaus.groovy.ast.expr.MethodCallExpression
+import org.codehaus.groovy.ast.expr.PropertyExpression
+import org.codehaus.groovy.ast.expr.VariableExpression
+import org.codehaus.groovy.ast.stmt.BlockStatement
+import org.codehaus.groovy.ast.stmt.ExpressionStatement
+import org.codehaus.groovy.ast.stmt.Statement
+import org.codehaus.groovy.control.CompilePhase
+import org.codehaus.groovy.control.SourceUnit
+import org.codehaus.groovy.syntax.SyntaxException
+import org.codehaus.groovy.transform.ASTTransformation
+import org.codehaus.groovy.transform.GroovyASTTransformation
+
+/**
+ * Inject file staging directives for file inputs in
+ * process definitions.
+ *
+ * Must be done during semantic analysis so that the
+ * necessary type information is available.
+ *
+ * @author Ben Sherman 
+ */
+@Slf4j
+@CompileStatic
+@GroovyASTTransformation(phase = CompilePhase.SEMANTIC_ANALYSIS)
+class ProcessInputPathXformImpl implements ASTTransformation {
+
+    @Override
+    void visit(ASTNode[] astNodes, SourceUnit unit) {
+        new DslCodeVisitor(unit).visitClass((ClassNode)astNodes[1])
+    }
+
+    @CompileStatic
+    static class DslCodeVisitor extends ClassCodeVisitorSupport {
+
+        private SourceUnit unit
+
+        DslCodeVisitor(SourceUnit unit) {
+            this.unit = unit
+        }
+
+        @Override
+        protected SourceUnit getSourceUnit() { unit }
+
+        @Override
+        void visitMethodCallExpression(MethodCallExpression methodCall) {
+            if( methodCall.objectExpression?.getText() != 'this' )
+                return
+
+            final methodName = methodCall.getMethodAsString()
+            if( methodName != 'process' )
+                return
+
+            final args = methodCall.arguments as ArgumentListExpression
+            final lastArg = args.expressions.size()>0 ? args.getExpression(args.expressions.size()-1) : null
+
+            if( lastArg !instanceof ClosureExpression ) {
+                syntaxError(lastArg, "Invalid process definition, possible syntax error")
+                return
+            }
+
+            final closure = (ClosureExpression)lastArg
+            final block = (BlockStatement)closure.code
+
+            List paramStatements = []
+            String currentLabel = null
+
+            for( final stmt : block.statements ) {
+                currentLabel = stmt.statementLabel ?: currentLabel
+                if( currentLabel != 'input' )
+                    continue
+                if( stmt !instanceof ExpressionStatement )
+                    continue
+                final stmtX = (ExpressionStatement)stmt
+                if( stmtX.expression !instanceof MethodCallExpression )
+                    continue
+                final call = (MethodCallExpression)stmtX.expression
+                if( call.methodAsString != '_typed_in_param' )
+                    continue
+                final paramArgs = (ArgumentListExpression)call.arguments
+                assert paramArgs.size() == 2
+                assert paramArgs[0] instanceof ConstantExpression
+                assert paramArgs[1] instanceof ClassExpression
+                final varName = ((ConstantExpression)paramArgs[0]).text
+                final varType = ((ClassExpression)paramArgs[1]).type
+
+                // infer staging directives via reflection
+                final var = new VariableExpression(varName, varType)
+                paramStatements.addAll( emitPathInputDecls(var, new TypeDef(var.type)) )
+            }
+
+            // prepend additional param statements
+            paramStatements.addAll(block.statements)
+            block.statements = paramStatements
+        }
+
+        protected List emitPathInputDecls( Expression expr, TypeDef typeDef ) {
+            List result = []
+            final type = typeDef.type
+
+            if( isPathType(typeDef) ) {
+                log.trace "inferring staging directive for path input: ${expr.text}"
+                final block = new BlockStatement()
+                block.addStatement( new ExpressionStatement(expr) )
+                final closure = new ClosureExpression(Parameter.EMPTY_ARRAY, block)
+                closure.variableScope = new VariableScope(block.variableScope)
+                result << new ExpressionStatement( callThisX( 'stageAs', args(closure) ) )
+            }
+            else if( isRecordType(type) ) {
+                for( final field : type.getDeclaredFields() ) {
+                    if( /* !field.isAccessible() || */ Modifier.isStatic(field.getModifiers()) )
+                        continue
+                    log.trace "inspecting record type ${type.name}: field=${field.name}, type=${field.type.name}"
+                    result.addAll( emitPathInputDecls(new PropertyExpression(expr, field.name), new TypeDef(field)) )
+                }
+            }
+
+            return result
+        }
+
+        protected boolean isRecordType(Class type) {
+            // NOTE: custom parser will be able to detect record types more elegantly
+            log.trace "is ${type.name} a record type? ${type.package?.name == ''}"
+            return type.package && type.package.name == ''
+        }
+
+        protected boolean isPathType(TypeDef typeDef) {
+            final type = typeDef.type
+
+            log.trace "is ${type.simpleName} a Path? ${type.name == 'java.nio.file.Path'}"
+            if( Path.isAssignableFrom(type) )
+                return true
+            if( Collection.isAssignableFrom(type) && typeDef.genericTypes ) {
+                final genericType = typeDef.genericTypes.first()
+                log.trace "is ${type.simpleName}<${genericType.simpleName}> a Collection? ${genericType.name == 'java.nio.file.Path'}"
+                return Path.isAssignableFrom(genericType)
+            }
+            return false
+        }
+
+        protected void syntaxError(ASTNode node, String message) {
+            int line = node.lineNumber
+            int coln = node.columnNumber
+            unit.addError( new SyntaxException(message,line,coln))
+        }
+
+    }
+
+    private static class TypeDef {
+        Class type
+        List genericTypes
+
+        TypeDef(ClassNode classNode) {
+            this.type = classNode.getPlainNodeReference().getTypeClass()
+            if( classNode.getGenericsTypes() )
+                this.genericTypes = classNode.getGenericsTypes().collect( el -> el.getType().getPlainNodeReference().getTypeClass() )
+        }
+
+        TypeDef(Field field) {
+            this.type = field.type
+            if( field.genericType instanceof ParameterizedType )
+                this.genericTypes = field.genericType.getActualTypeArguments() as List
+        }
+    }
+
+}
diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/ProcessOutputVisitor.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/ProcessOutputVisitor.groovy
new file mode 100644
index 0000000000..d5be7f7826
--- /dev/null
+++ b/modules/nextflow/src/main/groovy/nextflow/ast/ProcessOutputVisitor.groovy
@@ -0,0 +1,144 @@
+/*
+ * Copyright 2013-2024, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package nextflow.ast
+
+import groovy.transform.CompileStatic
+import org.codehaus.groovy.ast.ClassCodeVisitorSupport
+import org.codehaus.groovy.ast.expr.ArgumentListExpression
+import org.codehaus.groovy.ast.expr.Expression
+import org.codehaus.groovy.ast.expr.MapExpression
+import org.codehaus.groovy.ast.expr.MethodCallExpression
+import org.codehaus.groovy.ast.expr.VariableExpression
+import org.codehaus.groovy.ast.stmt.ExpressionStatement
+import org.codehaus.groovy.ast.stmt.Statement
+import org.codehaus.groovy.control.SourceUnit
+
+import static org.codehaus.groovy.ast.tools.GeneralUtils.*
+
+/**
+ * Extract unstaging directives from process output
+ *
+ * @author Ben Sherman 
+ */
+@CompileStatic
+class ProcessOutputVisitor extends ClassCodeVisitorSupport {
+
+    final SourceUnit sourceUnit
+
+    final List statements = []
+
+    private int evalCount = 0
+
+    private int pathCount = 0
+
+    ProcessOutputVisitor(SourceUnit unit) {
+        this.sourceUnit = unit
+    }
+
+    @Override
+    void visitMethodCallExpression(MethodCallExpression call) {
+        extractDirective(call)
+        super.visitMethodCallExpression(call)
+    }
+
+    void extractDirective(MethodCallExpression call) {
+        if( call.objectExpression?.text != 'this' )
+            return
+
+        if( call.arguments !instanceof ArgumentListExpression )
+            return
+
+        final name = call.methodAsString
+        final args = (ArgumentListExpression)call.arguments
+
+        /**
+         * env(name) -> _typed_out_env(name)
+         */
+        if( name == 'env' ) {
+            if( args.size() != 1 )
+                return
+
+            statements << makeDirective(
+                '_typed_out_env',
+                args[0]
+            )
+        }
+
+        /**
+         * eval(cmd) -> _typed_out_eval(key) { cmd }
+         *           -> eval(key)
+         */
+        else if( name == 'eval' ) {
+            if( args.size() != 1 )
+                return
+
+            final key = constX("nxf_out_eval_${evalCount++}".toString())
+
+            statements << makeDirective(
+                '_typed_out_eval',
+                key,
+                closureX(new ExpressionStatement(args[0]))
+            )
+
+            call.arguments = new ArgumentListExpression(key)
+        }
+
+        /**
+         * path(opts, pattern) -> _typed_out_path(opts, key) { pattern }
+         *                     -> path(key)
+         */
+        else if( name == 'path' ) {
+            def opts = null
+            def pattern
+            if( args.size() == 1 ) {
+                pattern = args[0]
+            }
+            else if( args.size() == 2 ) {
+                opts = args[0]
+                pattern = args[1]
+            }
+            else return
+
+            final key = constX("\$file${pathCount++}".toString())
+
+            statements << makeDirective(
+                '_typed_out_path',
+                opts ?: new MapExpression(),
+                key,
+                closureX(new ExpressionStatement(pattern))
+            )
+
+            call.arguments = new ArgumentListExpression(key)
+        }
+    }
+
+    Statement makeDirective(String name, Expression... args) {
+        new ExpressionStatement(
+            new MethodCallExpression(
+                new VariableExpression('this'),
+                name,
+                new ArgumentListExpression(args)
+            )
+        )
+    }
+
+    @Override
+    protected SourceUnit getSourceUnit() {
+        return sourceUnit
+    }
+
+}
diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy
index a528a10e97..706015a4d4 100644
--- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy
@@ -1530,8 +1530,13 @@ class TaskProcessor {
         final ctx = task.context
 
         // -- add input params to task context
-        for( int i = 0; i < inputs.size(); i++ )
+        for( int i = 0; i < inputs.size(); i++ ) {
+            final expectedType = inputs[i].type
+            final actualType = values[i].class
+            if( expectedType != null && !expectedType.isAssignableFrom(actualType) )
+                log.warn "[${safeTaskName(task)}] invalid argument type at index ${i} -- expected a ${expectedType.simpleName} but got a ${actualType.simpleName}"
             ctx.put(inputs[i].getName(), values[i])
+        }
 
         // -- resolve local variables
         for( def entry : inputs.getVariables() )
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessInput.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessInput.groovy
index bae5d200a8..7dade2ffcf 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessInput.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessInput.groovy
@@ -38,6 +38,11 @@ class ProcessInput implements Cloneable {
      */
     private String name
 
+    /**
+     * Parameter type which is used to validate task inputs
+     */
+    private Class type
+
     /**
      * Input channel which is created when the process is invoked
      * in a workflow.
@@ -49,14 +54,19 @@ class ProcessInput implements Cloneable {
      */
     private boolean iterator
 
-    ProcessInput(String name) {
+    ProcessInput(String name, Class type) {
         this.name = name
+        this.type = type
     }
 
     String getName() {
         return name
     }
 
+    Class getType() {
+        return type
+    }
+
     void bind(Object value) {
         this.channel = getInChannel(value)
     }
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessInputs.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessInputs.groovy
index 072e2d2509..11160ea9b7 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessInputs.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessInputs.groovy
@@ -55,8 +55,8 @@ class ProcessInputs implements List, Cloneable {
      */
     Object stdin
 
-    void addParam(String name) {
-        add(new ProcessInput(name))
+    void addParam(String name, Class type=null) {
+        add(new ProcessInput(name, type))
     }
 
     void addVariable(String name, Object value) {
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessOutput.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessOutput.groovy
index 694eb71007..a9452b5276 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessOutput.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessOutput.groovy
@@ -50,6 +50,12 @@ class ProcessOutput implements Cloneable {
      */
     private String name
 
+    /**
+     * Optional parameter type which is used to validate
+     * task outputs
+     */
+    private Class type
+
     /**
      * Optional channel topic which this output channel will
      * be sent to.
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessOutputs.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessOutputs.groovy
index 0e993edc07..e0fcb7497d 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessOutputs.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessOutputs.groovy
@@ -65,16 +65,12 @@ class ProcessOutputs implements List, Cloneable {
         env.add(name)
     }
 
-    String addEval(Object value) {
-        final key = "nxf_out_eval_${eval.size()}"
-        eval.put(key, value)
-        return key
+    void addEval(String name, Object value) {
+        eval.put(name, value)
     }
 
-    String addFile(ProcessFileOutput file) {
-        final key = "\$file${files.size()}"
+    void addFile(String key, ProcessFileOutput file) {
         files.put(key, file)
-        return key
     }
 
     List getNames() {
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ScriptParser.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ScriptParser.groovy
index 8931cf9b03..dd7287e49c 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ScriptParser.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ScriptParser.groovy
@@ -26,6 +26,7 @@ import nextflow.Session
 import nextflow.ast.NextflowDSL
 import nextflow.ast.NextflowXform
 import nextflow.ast.OpXform
+import nextflow.ast.ProcessInputPathXform
 import nextflow.exception.ScriptCompilationException
 import nextflow.extension.FilesEx
 import nextflow.file.FileHelper
@@ -124,6 +125,7 @@ class ScriptParser {
         config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowDSL))
         config.addCompilationCustomizers( new ASTTransformationCustomizer(NextflowXform))
         config.addCompilationCustomizers( new ASTTransformationCustomizer(OpXform))
+        config.addCompilationCustomizers( new ASTTransformationCustomizer(ProcessInputPathXform))
 
         if( session?.debug )
             config.debug = true
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessDsl.groovy b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessDsl.groovy
index 556e96ca7a..5b798a5bc3 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessDsl.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessDsl.groovy
@@ -54,6 +54,49 @@ class ProcessDsl extends ProcessBuilder {
         super(ownerScript, processName)
     }
 
+    /// TYPED INPUTS / OUTPUTS
+
+    void env(String name, Object source) {
+        inputs.addEnv(name, source)
+    }
+
+    void stageAs(Object source) {
+        inputs.addFile(new ProcessFileInput(source, null, true, [:]))
+    }
+
+    void stageAs(String stageAs, Object source) {
+        inputs.addFile(new ProcessFileInput(source, null, true, [stageAs: stageAs]))
+    }
+
+    void stdin(Object source) {
+        inputs.stdin = source
+    }
+
+    void _typed_in_param(String name, Class type) {
+        inputs.addParam(name, type)
+    }
+
+    void _typed_out_env(String name) {
+        outputs.addEnv(name)
+    }
+
+    void _typed_out_eval(String name, CharSequence cmd) {
+        outputs.addEval(name, cmd)
+    }
+
+    void _typed_out_path(Map opts=[:], String key, Object target) {
+        outputs.addFile(key, new ProcessFileOutput(target, true, opts))
+    }
+
+    void _typed_out_param(String name, Class type, Object target) {
+        final opts = [
+            name: name,
+            optional: type != null && Optional.isAssignableFrom(type),
+            type: type
+        ]
+        outputs.addParam(target, opts)
+    }
+
     /// INPUTS
 
     void _in_each(LazyVar var) {
@@ -192,10 +235,16 @@ class ProcessDsl extends ProcessBuilder {
         if( opts.emit )
             opts.name = opts.remove('emit')
 
-        final name = outputs.addEval(cmd)
+        final name = _out_eval0(cmd)
         outputs.addParam(new LazyEvalCall(name), opts)
     }
 
+    private String _out_eval0(CharSequence cmd) {
+        final name = "nxf_out_eval_${outputs.eval.size()}"
+        outputs.addEval(name, cmd)
+        return name
+    }
+
     void _out_file(Object target) {
         // note: check that is a String type to avoid to force
         // the evaluation of GString object to a string
@@ -226,7 +275,9 @@ class ProcessDsl extends ProcessBuilder {
     }
 
     private String _out_path0(Object target, boolean pathQualifier, Map opts) {
-        outputs.addFile(new ProcessFileOutput(target, pathQualifier, opts))
+        final key = "\$file${outputs.files.size()}"
+        outputs.addFile(key, new ProcessFileOutput(target, pathQualifier, opts))
+        return key
     }
 
     void _out_stdout(Map opts=[:]) {
@@ -263,7 +314,7 @@ class ProcessDsl extends ProcessBuilder {
                 target << new LazyEnvCall(name)
             }
             else if( item instanceof TokenEvalCall ) {
-                final name = outputs.addEval(item.val)
+                final name = _out_eval0(item.val)
                 target << new LazyEvalCall(name)
             }
             else if( item instanceof TokenFileCall ) {

From 8efcfc0e99719cc939a8b1af431443fbf4ab43f3 Mon Sep 17 00:00:00 2001
From: Ben Sherman 
Date: Thu, 28 Mar 2024 21:09:51 -0500
Subject: [PATCH 34/36] Update docs

Signed-off-by: Ben Sherman 
---
 docs/process.md | 163 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 163 insertions(+)

diff --git a/docs/process.md b/docs/process.md
index c700706e9d..4354987b14 100644
--- a/docs/process.md
+++ b/docs/process.md
@@ -821,6 +821,96 @@ In general, multiple input channels should be used to process *combinations* of
 
 See also: {ref}`channel-types`.
 
+(process-typed-inputs)=
+
+### Typed inputs
+
+:::{versionadded} 24.10.0
+:::
+
+Typed inputs are an alternative way to define process inputs with standard types. This approach has a number of benefits:
+
+- A typed input can validate the values that it receives at runtime and raise an error if there is a type mismatch.
+
+- Whereas a `path` input relies on a custom `arity` option in order to distinguish between a single file and a list of files, with typed inputs it is trivial: `Path` vs `List`.
+
+- Typed inputs enable the use of custom record types (i.e. defined using `@ValueObject` or `record`), which makes the code much easier to read and understand compared to `tuple` inputs.
+
+A typed input is simply a variable declaration, i.e. ` `. Here are some examples:
+
+```groovy
+input:
+int my_int
+String my_string
+Path my_file
+List my_files
+```
+
+In the above example:
+
+- `my_int` and `my_string` are treated like `val` inputs; they are defined in the process body as variables
+
+- `my_file` and `my_files` are treated like `path` inputs; they are defined as variables and their files are staged into the task directory
+
+One of the most important capabilities enabled by typed inputs is the use of custom record types. Here is an example:
+
+```groovy
+@ValueObject
+class Sample {
+  String id
+  List reads
+}
+
+process foo {
+  input:
+  Sample my_sample
+
+  // ...
+}
+```
+
+In this example, `Sample` is a record type with two members `id` and `reads`. The `Sample` input in process `foo` will be provided as a variable to the process body, where its members can be accessed as `my_sample.id` and `my_sample.reads`. Additionally, because `my_sample.reads` is a collection of files (given by its type `List`), it will be staged into the task directory like a `path` input.
+
+Environment variables and standard input can be defined using the new `env` and `stdin` directives. Building from the previous example:
+
+```groovy
+process foo {
+  env('SAMPLE_ID') { my_sample.id }
+  env('FIRST_READ_FILE') { my_sample.reads[0]?.name }
+  stdin { my_sample.reads[0] }
+
+  input:
+  Sample my_sample
+
+  // ...
+}
+```
+
+In the above example:
+
+- The sample id will be exported to the `SAMPLE_ID` variable in the task environment
+- The name of the first sample read file will be exported to the `FIRST_READ_FILE` variable in the task environment
+- The contents of the first sample read file will be provided as standard input to the task
+
+By default, file inputs are automatically inferred from the types and staged into the task directory. Alternatively, the `stageAs` directive can be used to stage files under a different name, similar to using the `name` or `stageAs` option with a `path` input. For example:
+
+```groovy
+process foo {
+  stageAs('*.fastq') { my_sample.reads }
+
+  input:
+  Sample my_sample
+
+  // ...
+}
+```
+
+In this case, `my_sample.reads` will be staged as `*.fastq`, overriding the default behavior.
+
+:::{note}
+While the `env`, `stageAs`, and `stdin` directives are provided as a convenience, it is usually easier to simply rely on the default file staging behavior, and to use the input variables directly in the task script.
+:::
+
 (process-output)=
 
 ## Outputs
@@ -1202,6 +1292,79 @@ The following options are available for all process outputs:
 
 : Defines the {ref}`channel topic ` to which the output will be sent.
 
+(process-typed-outputs)=
+
+### Typed outputs
+
+:::{versionadded} 24.10.0
+:::
+
+Typed outputs are an alternative way to define process outputs with standard types. This approach has a number of benefits:
+
+- A typed output clearly describes the expected structure of the output, which makes it easier to use the output in downstream operations.
+
+- Whereas a `path` output relies on a custom `arity` option in order to distinguish between a single file and a list of files, with typed outputs it is trivial: `Path` vs `List`.
+
+- Typed outputs enable the use of custom record types (i.e. defined using `@ValueObject` or `record`), which makes the code much easier to read and understand compared to `tuple` outputs.
+
+A typed output is simply a variable declaration with an optional assignment, i.e. `  [= ]`. Here are some examples:
+
+```groovy
+output:
+int my_int
+String my_string = my_input
+Path my_file = path('file1.txt')
+List my_files = path('*.txt')
+```
+
+In the above example:
+
+- `my_int` and `my_string` are treated like `val` outputs; they are assigned to the variables `my_int` and `my_input`, which are expected to be defined in the process body
+
+- `my_file` and `my_files` are treated like `path` outputs; they are assigned to a file or list of files based on a matching pattern using the `path()` method
+
+- The output variable names correspond to the `emit` option for process outputs
+
+One of the most important capabilities enabled by typed outputs is the use of custom record types. Here is an example:
+
+```groovy
+@ValueObject
+class Sample {
+  String id
+  List reads
+}
+
+process foo {
+  input:
+  String id
+
+  output:
+  Sample my_sample = new Sample(id, path('*.fastq'))
+
+  // ...
+}
+```
+
+In this example, `Sample` is a record type with two members `id` and `reads`. The `Sample` output will be constructed from the `id` input variable and the collection of task output files matching the pattern `*.fastq`.
+
+In addition to the `path()` method, there are also the `env()`, `eval()`, and `stdout()` methods for extracting environment variables, eval commands, and standard output from the task environment. For example:
+
+```groovy
+process foo {
+  // ...
+
+  output:
+  String my_env = env('MY_VAR')
+  String my_eval = eval('bash --version')
+  String my_stdout = stdout()
+  List my_tuple = [ env('MY_VAR'), eval('bash --version'), stdout() ]
+
+  // ...
+}
+```
+
+As shown in the above examples, output values can be any expression, including lists, maps, records, and even function calls.
+
 ## When
 
 The `when` block allows you to define a condition that must be satisfied in order to execute the process. The condition can be any expression that returns a boolean value.

From 8a3a8277eecead8c5c73691256b0804fa25dcd91 Mon Sep 17 00:00:00 2001
From: Ben Sherman 
Date: Thu, 28 Mar 2024 21:19:17 -0500
Subject: [PATCH 35/36] Fix error with legacy syntax

Signed-off-by: Ben Sherman 
---
 .../src/main/groovy/nextflow/ast/NextflowDSLImpl.groovy       | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSLImpl.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSLImpl.groovy
index 328414d51e..75c6bc3df4 100644
--- a/modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSLImpl.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSLImpl.groovy
@@ -890,7 +890,7 @@ class NextflowDSLImpl implements ASTTransformation {
             )
         }
 
-        private static final VALID_INPUT_METHODS = ['val','env','file','path','stdin','each','tuple']
+        private static final List VALID_INPUT_METHODS = ['val','env','file','path','stdin','each','tuple']
 
         protected void convertInputMethod( MethodCallExpression methodCall ) {
             final methodName = methodCall.getMethodAsString()
@@ -971,7 +971,7 @@ class NextflowDSLImpl implements ASTTransformation {
             return visitor.statements
         }
 
-        private static final VALID_OUTPUT_METHODS = ['val','env','eval','file','path','stdout','tuple']
+        private static final List VALID_OUTPUT_METHODS = ['val','env','eval','file','path','stdout','tuple']
 
         protected void convertOutputMethod( MethodCallExpression methodCall ) {
             final methodName = methodCall.getMethodAsString()

From 1cd6fce697a2c6eb99de4629f0495eba07e118cb Mon Sep 17 00:00:00 2001
From: Ben Sherman 
Date: Thu, 28 Mar 2024 23:58:29 -0500
Subject: [PATCH 36/36] Rename CombineManyOp -> MergeWithEachOp

Signed-off-by: Ben Sherman 
---
 ...neManyOp.groovy => MergeWithEachOp.groovy} |  4 +-
 .../nextflow/processor/TaskProcessor.groovy   | 44 +++++++++----------
 .../groovy/nextflow/script/ProcessDef.groovy  |  4 +-
 3 files changed, 25 insertions(+), 27 deletions(-)
 rename modules/nextflow/src/main/groovy/nextflow/extension/{CombineManyOp.groovy => MergeWithEachOp.groovy} (97%)

diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/CombineManyOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/MergeWithEachOp.groovy
similarity index 97%
rename from modules/nextflow/src/main/groovy/nextflow/extension/CombineManyOp.groovy
rename to modules/nextflow/src/main/groovy/nextflow/extension/MergeWithEachOp.groovy
index a76d47e7ea..926942d3ac 100644
--- a/modules/nextflow/src/main/groovy/nextflow/extension/CombineManyOp.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/extension/MergeWithEachOp.groovy
@@ -35,7 +35,7 @@ import nextflow.Channel
  */
 @Slf4j
 @CompileStatic
-class CombineManyOp {
+class MergeWithEachOp {
 
     private List sources
 
@@ -65,7 +65,7 @@ class CombineManyOp {
 
     private transient List combinations
 
-    CombineManyOp(List sources, List iterators) {
+    MergeWithEachOp(List sources, List iterators) {
         this.sources = sources
         this.iterators = iterators
         this.queues = sources.collect( ch -> [] )
diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy
index 706015a4d4..cf02149172 100644
--- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy
@@ -663,7 +663,7 @@ class TaskProcessor {
             task.cached = true
             session.notifyTaskCached(new StoredTaskHandler(task))
 
-            // -- now bind the results
+            // -- now emit the results
             finalizeTask0(task)
             return true
         }
@@ -748,7 +748,7 @@ class TaskProcessor {
             if( entry )
                 session.notifyTaskCached(new CachedTaskHandler(task,entry.trace))
 
-            // -- now bind the results
+            // -- now emit the results
             finalizeTask0(task)
             return true
         }
@@ -1162,49 +1162,48 @@ class TaskProcessor {
     }
 
     /**
-     * Bind the expected output files to the corresponding output channels
-     * @param processor
+     * Emit the expected outputs to the corresponding output channels
      */
-    synchronized protected void bindOutputs( TaskRun task ) {
+    synchronized protected void emitOutputs( TaskRun task ) {
 
-        // bind the output
+        // -- emit the output
         if( isFair0 ) {
-            fairBindOutputs0(task.outputs, task)
+            fairEmitOutputs0(task.outputs, task)
         }
         else {
-            bindOutputs0(task.outputs)
+            emitOutputs0(task.outputs)
         }
 
-        // -- finally prints out the task output when 'debug' is true
+        // -- finally print the task output when 'debug' is true
         if( task.config.debug ) {
             task.echoStdout(session)
         }
     }
 
-    protected void fairBindOutputs0(List emissions, TaskRun task) {
+    protected void fairEmitOutputs0(List emissions, TaskRun task) {
         synchronized (isFair0) {
             // decrement -1 because tasks are 1-based
             final index = task.index-1
-            // store the task emission values in a buffer
+            // store the task output values in a buffer
             fairBuffers[index-currentEmission] = emissions
-            // check if the current task index matches the expected next emission index
+            // check if the current task index matches the expected next output index
             if( currentEmission == index ) {
                 while( emissions!=null ) {
-                    // bind the emission values
-                    bindOutputs0(emissions)
+                    // emit the output values
+                    emitOutputs0(emissions)
                     // remove the head and try with the following
                     fairBuffers.remove(0)
-                    // increase the index of the next emission
+                    // increase the index of the next output
                     currentEmission++
-                    // take the next emissions 
+                    // take the next output
                     emissions = fairBuffers[0]
                 }
             }
         }
     }
 
-    protected void bindOutputs0(List outputs) {
-        // -- bind out the collected values
+    protected void emitOutputs0(List outputs) {
+        // -- emit the output values
         for( int i = 0; i < config.getOutputs().size(); i++ ) {
             final param = config.getOutputs()[i]
             final value = outputs[i]
@@ -1780,7 +1779,7 @@ class TaskProcessor {
 
     /**
      * Finalize the task execution, checking the exit status
-     * and binding output values accordingly
+     * and emitting output values accordingly
      *
      * @param task The {@code TaskRun} instance to finalize
      */
@@ -1832,17 +1831,16 @@ class TaskProcessor {
 
     /**
      * Finalize the task execution, checking the exit status
-     * and binding output values accordingly
+     * and emitting output values accordingly
      *
      * @param task The {@code TaskRun} instance to finalize
-     * @param producedFiles The map of files to be bind the outputs
      */
     private void finalizeTask0( TaskRun task ) {
         log.trace "Finalize process > ${safeTaskName(task)}"
 
-        // -- bind output (files)
+        // -- emit outputs
         if( task.canBind ) {
-            bindOutputs(task)
+            emitOutputs(task)
             publishOutputs(task)
         }
 
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy
index 660a386989..073080c451 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy
@@ -25,7 +25,7 @@ import nextflow.NF
 import nextflow.Session
 import nextflow.exception.ScriptRuntimeException
 import nextflow.extension.CH
-import nextflow.extension.CombineManyOp
+import nextflow.extension.MergeWithEachOp
 import nextflow.script.dsl.ProcessConfigBuilder
 
 /**
@@ -167,7 +167,7 @@ class ProcessDef extends BindableDef implements IterableDef, ChainableDef {
         }
 
         final iterators = (0.. declaredInputs[i].isIterator() )
-        return CH.getReadChannel(new CombineManyOp(declaredInputs.getChannels(), iterators).apply())
+        return CH.getReadChannel(new MergeWithEachOp(declaredInputs.getChannels(), iterators).apply())
     }
 
     private void collectOutputs(boolean singleton) {