Add topics to the template + update linting

CHANGELOG.md

@@ -16,6 +16,7 @@
 - ignore files in gitignore also for pipeline_if_empty_null lint test ([#3722](https://github.com/nf-core/tools/pull/3722))
 - do not check pytest_modules.yml file, deprecating ([#3748](https://github.com/nf-core/tools/pull/3748))
 - Use the org from the .nf-core.yml when linting manifest name and homePage. ([#3767](https://github.com/nf-core/tools/pull/3767))
+- Add `topics` to the template + update linting ([#3779](https://github.com/nf-core/tools/pull/3779))
 
 ### Modules
 

nf_core/components/nfcore_component.py

@@ -253,7 +253,7 @@ def get_outputs_from_main_nf(self):
                 return outputs
             output_data = data.split("output:")[1].split("when:")[0]
             regex_emit = r"emit:\s*([^)\s,]+)"
-            regex_elements = r"\b(val|path|env|stdout)\s*(\(([^)]+)\)|\s*([^)\s,]+))"
+            regex_elements = r"\b(val|path|env|stdout|eval)\s*(\(([^)]+)\)|\s*([^)\s,]+))"
             for line in output_data.split("\n"):
                 match_emit = re.search(regex_emit, line)
                 matches_elements = re.finditer(regex_elements, line)

nf_core/module-template/main.nf

@@ -65,7 +65,12 @@ process {{ component_name_underscore|upper }} {
     {{ 'tuple val(meta), path("*")' if has_meta else 'path "*"' }}, emit: output
     {%- endif %}
     {%- endif %}
-    path "versions.yml"           , emit: versions
+    {% if not_empty_template -%}
+    // TODO nf-core: Update the command here to obtain the version number of the software used in this module
+    // TODO nf-core: If multiple software packages are used in this module then each MUST be added here
+    //               by copying the line below and replacing the current tool with the extra tool(s)
+    {%- endif %}
+    tuple val("${task.process}"), val('{{ component }}'), eval("{{ component }} --version"), topic: versions, emit: versions1
 
     when:
     task.ext.when == null || task.ext.when
@@ -111,11 +116,6 @@ process {{ component_name_underscore|upper }} {
         $bam
         {%- endif %}
     {%- endif %}
-
-    cat <<-END_VERSIONS > versions.yml
-    "${task.process}":
-        {{ component }}: \$({{ component }} --version)
-    END_VERSIONS
     """
 
     stub:
@@ -146,10 +146,5 @@ process {{ component_name_underscore|upper }} {
     touch ${prefix}.bam
     {%- endif %}
     {%- endif %}
-
-    cat <<-END_VERSIONS > versions.yml
-    "${task.process}":
-        {{ component }}: \$({{ component }} --version)
-    END_VERSIONS
     """
 }

nf_core/module-template/meta.yml

@@ -46,13 +46,16 @@ output:
             - edam: "http://edamontology.org/format_2572" # BAM
             - edam: "http://edamontology.org/format_2573" # CRAM
             - edam: "http://edamontology.org/format_3462" # SAM
-  versions:
-    - "versions.yml":
-        type: file
-        description: File containing software versions
-        pattern: "versions.yml"
-        ontologies:
-          - edam: "http://edamontology.org/format_3750" # YAML
+  versions1:
+    - - "${task.process}":
+          type: string
+          description: The name of the process
+      - "{{ component }}":
+          type: string
+          description: The name of the tool
+      - "{{ component }} --version":
+          type: string
+          description: The version of the tool
 
 authors:
   - "{{ author }}"

nf_core/modules/lint/main_nf.py

@@ -36,14 +36,15 @@ def main_nf(
     * The module has a process label and it is among
       the standard ones.
     * If a ``meta`` map is defined as one of the modules
-      inputs it should be defined as one of the outputs,
+      inputs it should be defined as one of the emits,
       and be correctly configured in the ``saveAs`` function.
     * The module script section should contain definitions
       of ``software`` and ``prefix``
     """
 
     inputs: list[str] = []
-    outputs: list[str] = []
+    emits: list[str] = []
+    topics: list[str] = []
 
     # Check if we have a patch file affecting the 'main.nf' file
     # otherwise read the lines directly from the module
@@ -128,8 +129,9 @@ def main_nf(
                 line = joint_tuple
             inputs.extend(_parse_input(module, line))
         if state == "output" and not _is_empty(line):
-            outputs += _parse_output(module, line)
-            outputs = list(set(outputs))  # remove duplicate 'meta's
+            emits += _parse_output_emits(module, line)
+            emits = list(set(emits))  # remove duplicate 'meta's
+            topics += _parse_output_topics(module, line)
         if state == "when" and not _is_empty(line):
             when_lines.append(line)
         if state == "script" and not _is_empty(line):
@@ -140,7 +142,7 @@ def main_nf(
             exec_lines.append(line)
 
     # Check that we have required sections
-    if not len(outputs):
+    if not len(emits):
         module.failed.append(("main_nf_script_outputs", "No process 'output' block found", module.main_nf))
     else:
         module.passed.append(("main_nf_script_outputs", "Process 'output' block found", module.main_nf))
@@ -175,8 +177,8 @@ def main_nf(
     if inputs:
         if "meta" in inputs:
             module.has_meta = True
-            if outputs:
-                if "meta" in outputs:
+            if emits:
+                if "meta" in emits:
                     module.passed.append(
                         ("main_nf_meta_output", "'meta' map emitted in output channel(s)", module.main_nf)
                     )
@@ -186,13 +188,22 @@ def main_nf(
                     )
 
     # Check that a software version is emitted
-    if outputs:
-        if "versions" in outputs:
-            module.passed.append(("main_nf_version_emitted", "Module emits software version", module.main_nf))
+    if topics:
+        if "versions" in topics:
+            module.passed.append(("main_nf_version_topic", "Module emits software versions as topic", module.main_nf))
         else:
-            module.warned.append(("main_nf_version_emitted", "Module does not emit software version", module.main_nf))
+            module.failed.append(("main_nf_version_topic", "Module does not emit software versions as topic", module.main_nf))
 
-    return inputs, outputs
+    if emits:
+        topic_versions_amount = sum(1 for t in topics if t == "versions")
+        emit_versions_amount = sum(1 for e in emits if e.startswith("versions"))
+        if topic_versions_amount == emit_versions_amount:
+            module.passed.append(("main_nf_version_emit", "Module emits each software version", module.main_nf))
+        else:
+            module.failed.append(("main_nf_version_emit", "Module does not have an `emit:` and `topic:` for each software version", module.main_nf))
+
+
+    return inputs, emits
 
 
 def check_script_section(self, lines):
@@ -202,12 +213,6 @@ def check_script_section(self, lines):
     """
     script = "".join(lines)
 
-    # check that process name is used for `versions.yml`
-    if re.search(r"\$\{\s*task\.process\s*\}", script):
-        self.passed.append(("main_nf_version_script", "Process name used for versions.yml", self.main_nf))
-    else:
-        self.warned.append(("main_nf_version_script", "Process name not used for versions.yml", self.main_nf))
-
     # check for prefix (only if module has a meta map as input)
     if self.has_meta:
         if re.search(r"\s*prefix\s*=\s*task.ext.prefix", script):
@@ -607,15 +612,30 @@ def _parse_input(self, line_raw):
     return inputs
 
 
-def _parse_output(self, line):
+def _parse_output_emits(self, line):
     output = []
     if "meta" in line:
         output.append("meta")
-    if "emit:" not in line:
+    emit_regex = re.search(r"^.*emit:\s*([^,\s]*)", line)
+    if not emit_regex:
         self.failed.append(("missing_emit", f"Missing emit statement: {line.strip()}", self.main_nf))
     else:
-        output.append(line.split("emit:")[1].strip())
+        output.append(emit_regex.group(1).strip())
+    return output
 
+def _parse_output_topics(self,line):
+    output = []
+    if "meta" in line:
+        output.append("meta")
+    topic_regex = re.search(r"^.*topic:\s*([^,\s]*)", line)
+    if topic_regex:
+        topic_name = topic_regex.group(1).strip()
+        output.append(topic_name)
+        if topic_name == "versions":
+            if not re.search(r'tuple\s+val\("\${\s*task\.process\s*}"\),\s*val\(.*\),\s*eval\(.*\)', line):
+                self.failed.append(("wrong_version_output", 'Versions topic output is not correctly formatted, expected `tuple val("${task.process}"), val(\'<tool>\'), eval("<version_command>")`', self.main_nf))
+            if not re.search(r'emit:\s*versions\d+', line):
+                self.failed.append(("wrong_version_emit", 'Version emit should follow the format `versions<version_number>`, e.g.: `versions1`, `versions2`', self.main_nf))
     return output
 
 

nf_core/subworkflow-template/main.nf

@@ -14,9 +14,6 @@ workflow {{ component_name_underscore|upper }} {
     ch_bam // channel: [ val(meta), [ bam ] ]
 
     main:
-
-    ch_versions = Channel.empty()
-
     // TODO nf-core: substitute modules here for the modules of your subworkflow
 
     SAMTOOLS_SORT ( ch_bam )
@@ -30,6 +27,4 @@ workflow {{ component_name_underscore|upper }} {
     bam      = SAMTOOLS_SORT.out.bam           // channel: [ val(meta), [ bam ] ]
     bai      = SAMTOOLS_INDEX.out.bai          // channel: [ val(meta), [ bai ] ]
     csi      = SAMTOOLS_INDEX.out.csi          // channel: [ val(meta), [ csi ] ]
-
-    versions = ch_versions                     // channel: [ versions.yml ]
 }