PublishDir outside process definition

[hkaspersen]

Is it possible to define the published output of a process outside the process itself?
This would be very useful if the same process was shared between pipelines, but didn't have the same output.

E.g. something like this:

process PROCESS_X {
    input:
    file("*")

    output: 
    file("*")

    script:
    ```
    something something &> process.log
    ```
}


workflow {
    PROCESS_X(publishDir "${params.out_dir}/logs", pattern: "*.log", mode: "copy")
}

[Kibubu]

You could include the process from a module and overwrite the parameters.

FIles:

.
├── main.nf
└── module.nf

main.nf

params.publishDir="asdf"
include { p } from './module'
include { p as p2 } from './module' params(publishDir: "qwer")

def getPath(){
    "foobar"
}

include { p as p3 } from './module' params(publishDir: getPath())

workflow{
p()
println p.dump()
// > ... processName=p ... publishDir:[[path:asdf]]]...

p2()
println p2.dump()
// > ... processName=p2 ... publishDir:[[path:qwer]]]...

p3()
println p3.dump()
// > ... processName=p3 ... publishDir:[[path:foobar]]]...
}

module.nf

process p{
publishDir params.publishDir
"""
"""
}

Docs

[hkaspersen]

I did not know you could provide the publishDir parameters like that, thanks!

[Kibubu]

be sure to read the docs on params as those can be quite tricky and imo not intuitive if params is used inside of module and host.

This post now became way to complex. But it lists all the possible combinations and the tricky parts are marked with EXPECTED

module.nf

params.publishDir = "module"
params.debug = "module"

process p{
publishDir params.publishDir
input:
  val label
output:
  tuple val(label), val(params.publishDir), val(params.debug)

  """
  """
}

includeAfterParams.nf

params.publishDir = "host"
include { p as params} from './module'
include { p as paramsWithParams} from './module' params(publishDir: "overwrite")
include { p as paramsWithParamsAndDebug} from './module' params(publishDir: "overwrite", debug:"overwrite")
include { p as paramsWithAddParams} from './module' addParams(publishDir: "overwrite")
include { p as paramsWithAddParamsAndDebug} from './module' addParams(publishDir: "overwrite", debug:"overwrite")

workflow {
  params("")
  params.out.view()
  // > [, host, module]
  paramsWithParams("params")
  paramsWithParams.out.view()
  // > [params, overwrite, module]
  paramsWithParamsAndDebug("paramsAndDebug")
  paramsWithParamsAndDebug.out.view()
  // > [paramsAndDebug, overwrite, overwrite]
  paramsWithAddParams("addParams")
  paramsWithAddParams.out.view()
  // > [addParams, host, module] **EXPECTED [addParams, overwrite, module]**
  paramsWithAddParamsAndDebug("addParamsAndDebug")
  paramsWithAddParamsAndDebug.out.view()
  // > [addParamsAndDebug, host, overwrite] **EXPECTED [addParamsAndDebug, overwrite, overwrite]**
}

includeBeforeParams.nf

include { p as params} from './module'
include { p as paramsWithParams} from './module' params(publishDir: "overwrite")
include { p as paramsWithParamsAndDebug} from './module' params(publishDir: "overwrite", debug:"overwrite")
include { p as paramsWithAddParams} from './module' addParams(publishDir: "overwrite")
include { p as paramsWithAddParamsAndDebug} from './module' addParams(publishDir: "overwrite", debug:"overwrite")

params.publishDir = "host"

workflow {
  params("")
  params.out.view()
  // > [, module, module]
  paramsWithParams("params")
  paramsWithParams.out.view()
  // > [params, overwrite, module]
  paramsWithParamsAndDebug("paramsAndDebug")
  paramsWithParamsAndDebug.out.view()
  // > [paramsAndDebug, overwrite, overwrite]
  paramsWithAddParams("addParams")
  paramsWithAddParams.out.view()
  // > [addParams, overwrite, module]
  paramsWithAddParamsAndDebug("addParamsAndDebug")
  paramsWithAddParamsAndDebug.out.view()
  // > [addParamsAndDebug, overwrite, overwrite]
}

includeBeforeAndAfterParams.nf

include { p as beforeParams} from './module'
include { p as beforeParamsWithParams} from './module' params(publishDir: "overwrite")
include { p as beforeParamsWithParamsAndDebug} from './module' params(publishDir: "overwrite", debug:"overwrite")
include { p as beforeParamsWithAddParams} from './module' addParams(publishDir: "overwrite")
include { p as beforeParamsWithAddParamsAndDebug} from './module' addParams(publishDir: "overwrite", debug:"overwrite")

params.publishDir = "host"
include { p as afterParams} from './module'
include { p as afterParamsWithParams} from './module' params(publishDir: "overwrite")
include { p as afterParamsWithParamsAndDebug} from './module' params(publishDir: "overwrite", debug:"overwrite")
include { p as afterParamsWithAddParams} from './module' addParams(publishDir: "overwrite")
include { p as afterParamsWithAddParamsAndDebug} from './module' addParams(publishDir: "overwrite", debug:"overwrite")

workflow {
  beforeParams("before")
  beforeParams.out.view()
  // > [before, module, module]
  beforeParamsWithParams("beforeParams")
  beforeParamsWithParams.out.view()
  // > [beforeParams, overwrite, module]
  beforeParamsWithParamsAndDebug("beforeParamsAndDebug")
  beforeParamsWithParamsAndDebug.out.view()
  // > [beforeParamsAndDebug, overwrite, overwrite]
  beforeParamsWithAddParams("beforeAddParams")
  beforeParamsWithAddParams.out.view()
  // > [beforeAddParams, overwrite, module]
  beforeParamsWithAddParamsAndDebug("beforeAddParamsAndDebug")
  beforeParamsWithAddParamsAndDebug.out.view()
  // > [beforeAddParamsAndDebug, overwrite, overwrite]

  afterParams("after")
  afterParams.out.view()
  // > [after, module, module]  **EXPECTED [after, host, module]**
  afterParamsWithParams("afterParams")
  afterParamsWithParams.out.view()
  // > [afterParams, overwrite, module]
  afterParamsWithParamsAndDebug("afterParamsAndDebug")
  afterParamsWithParamsAndDebug.out.view()
  // > [afterParamsAndDebug, overwrite, overwrite]
  afterParamsWithAddParams("afterAddParams")
  afterParamsWithAddParams.out.view()
  // > [afterAddParams, host, module] **EXPECTED [afterAddParams, overwrite, module]**
  afterParamsWithAddParamsAndDebug("afterAddParamsAndDebug")
  afterParamsWithAddParamsAndDebug.out.view()
  // > [afterAddParamsAndDebug, host, overwrite] **EXPECTED [afterAddParamsAndDebug, overwrite, overwrite]**
}

[GordianDziwis]

Looks like the documentation is wrong:

The option addParams can be used to extend the module parameters without affecting the external scope.

Should be:

The option addParams can be used to extend the module parameters without overwriting parametes from the external scope.

Then it would be all consistent with your observations.

@Kibubu did you create an issue?

[Kibubu]

This should be the accepted answer. Way simpler

process PROCESS_X {
    publishDir '/process/dir'
    input:
    val publishDir
    script:
    task.publishDir = publishDir
    """
    """
}
workflow {
    PROCESS_X([[path:"/argument/dir", pattern:"*.log", mode:"copy"]])
}

[bentsherman]

See #1540 for related discussion about making a publish operator than can be used anywhere in your workflow.