Merge pull request #3779 from nvnieuwk/feat/topics

Add `topics` to the template + update linting

Author: nvnieuwk

CHANGELOG.md

@@ -50,6 +50,7 @@
 - ignore files in gitignore also for pipeline_if_empty_null lint test ([#3722](https://github.com/nf-core/tools/pull/3722))
 - do not check pytest_modules.yml file, deprecating ([#3748](https://github.com/nf-core/tools/pull/3748))
 - Use the org from the .nf-core.yml when linting manifest name and homePage. ([#3767](https://github.com/nf-core/tools/pull/3767))
+- Add `topics` to the template + update linting ([#3779](https://github.com/nf-core/tools/pull/3779))
 - Use the org from .nf-core.yml when linting multiqc_config report_comment ([#3800](https://github.com/nf-core/tools/pull/3800))
 - Linting of patched subworkflows ([#3755](https://github.com/nf-core/tools/pull/3755))
 - Add link to modules and subworkflows linting error docs ([#3818](https://github.com/nf-core/tools/pull/3818))

nf_core/components/create.py

@@ -528,21 +528,31 @@ def generate_meta_yml_file(self) -> None:
         with open(self.file_paths["meta.yml"]) as fh:
             meta_yml: ruamel.yaml.comments.CommentedMap = yaml.load(fh)
 
-        versions: dict[str, list[dict[str, dict]]] = {
+        versions: dict[str, list | dict] = {
+            f"versions_{self.component}": [
+                [
+                    {"${task.process}": {"type": "string", "description": "The name of the process"}},
+                    {f"{self.component}": {"type": "string", "description": "The name of the tool"}},
+                    {
+                        f"{self.component} --version": {"type": "string", "description": "The version of the tool"},
+                    },
+                ]
+            ]
+        }
+
+        versions_topic: dict[str, list | dict] = {
             "versions": [
-                {
-                    "versions.yml": {
-                        "type": "file",
-                        "description": "File containing software versions",
-                        "pattern": "versions.yml",
-                        "ontologies": [
-                            ruamel.yaml.comments.CommentedMap({"edam": "http://edamontology.org/format_3750"})
-                        ],
-                    }
-                }
+                [
+                    {"process": {"type": "string", "description": "The process the versions were collected from"}},
+                    {
+                        "tool": {"type": "string", "description": "The tool name the version was collected for"},
+                    },
+                    {
+                        "version": {"type": "string", "description": "The version of the tool"},
+                    },
+                ]
             ]
         }
-        versions["versions"][0]["versions.yml"]["ontologies"][0].yaml_add_eol_comment("YAML", "edam")
 
         if self.not_empty_template:
             meta_yml.yaml_set_comment_before_after_key(
@@ -557,8 +567,11 @@ def generate_meta_yml_file(self) -> None:
             meta_yml["output"].yaml_set_start_comment(
                 "### TODO nf-core: Add a description of all of the variables used as output", indent=2
             )
+            meta_yml["topics"].yaml_set_start_comment(
+                "### TODO nf-core: Add a description of all of the variables used as topics", indent=2
+            )
 
-            if hasattr(self, "inputs"):
+            if hasattr(self, "inputs") and len(self.inputs) > 0:
                 inputs_array: list[dict | list[dict]] = []
                 for i, (input_name, ontologies) in enumerate(self.inputs.items()):
                     channel_entry: dict[str, dict] = {
@@ -607,7 +620,7 @@ def generate_meta_yml_file(self) -> None:
                 meta_yml["input"][0]["bam"]["ontologies"][1].yaml_add_eol_comment("CRAM", "edam")
                 meta_yml["input"][0]["bam"]["ontologies"][2].yaml_add_eol_comment("SAM", "edam")
 
-            if hasattr(self, "outputs"):
+            if hasattr(self, "outputs") and len(self.outputs) > 0:
                 outputs_dict: dict[str, list | dict] = {}
                 for i, (output_name, ontologies) in enumerate(self.outputs.items()):
                     channel_contents: list[list[dict] | dict] = []
@@ -668,6 +681,8 @@ def generate_meta_yml_file(self) -> None:
                 meta_yml["output"]["bam"][0]["*.bam"]["ontologies"][2].yaml_add_eol_comment("SAM", "edam")
                 meta_yml["output"].update(versions)
 
+            meta_yml["topics"] = versions_topic
+
         else:
             input_entry: list[dict] = [
                 {"input": {"type": "file", "description": "", "pattern": "", "ontologies": [{"edam": ""}]}}
@@ -690,6 +705,7 @@ def generate_meta_yml_file(self) -> None:
                 meta_yml["input"] = input_entry
                 meta_yml["output"] = {"output": output_entry}
             meta_yml["output"].update(versions)
+            meta_yml["topics"] = versions_topic
 
         with open(self.file_paths["meta.yml"], "w") as fh:
             yaml.dump(meta_yml, fh)

nf_core/components/nfcore_component.py

@@ -253,7 +253,7 @@ def get_outputs_from_main_nf(self):
                 return outputs
             output_data = data.split("output:")[1].split("when:")[0]
             regex_emit = r"emit:\s*([^)\s,]+)"
-            regex_elements = r"\b(val|path|env|stdout)\s*(\(([^)]+)\)|\s*([^)\s,]+))"
+            regex_elements = r"\b(val|path|env|stdout|eval)\s*(\(([^)]+)\)|\s*([^)\s,]+))"
             for line in output_data.split("\n"):
                 match_emit = re.search(regex_emit, line)
                 matches_elements = re.finditer(regex_elements, line)
@@ -294,3 +294,41 @@ def get_outputs_from_main_nf(self):
                     pass
             log.debug(f"Found {len(outputs)} outputs in {self.main_nf}")
             self.outputs = outputs
+
+    def get_topics_from_main_nf(self) -> None:
+        with open(self.main_nf) as f:
+            data = f.read()
+        if self.component_type == "modules":
+            topics: dict[str, list[dict[str, dict] | list[dict[str, dict[str, str]]]]] = {}
+            # get topic name from main.nf after "output:". the names are always after "topic:"
+            if "output:" not in data:
+                log.debug(f"Could not find any outputs in {self.main_nf}")
+                self.topics = topics
+                return
+            output_data = data.split("output:")[1].split("when:")[0]
+            regex_topic = r"topic:\s*([^)\s,]+)"
+            regex_elements = r"\b(val|path|env|stdout|eval)\s*(\(([^)]+)\)|\s*([^)\s,]+))"
+            for line in output_data.split("\n"):
+                match_topic = re.search(regex_topic, line)
+                matches_elements = re.finditer(regex_elements, line)
+                if not match_topic:
+                    continue
+                channel_elements: list[dict[str, dict]] = []
+                topic_name = match_topic.group(1)
+                if topic_name in topics:
+                    continue
+                topics[match_topic.group(1)] = []
+                for count, match_element in enumerate(matches_elements, start=1):
+                    output_val = None
+                    if match_element.group(3):
+                        output_val = match_element.group(3)
+                    elif match_element.group(4):
+                        output_val = match_element.group(4)
+                    if output_val:
+                        channel_elements.append({f"value{count}": {}})
+                if len(channel_elements) == 1:
+                    topics[match_topic.group(1)].append(channel_elements[0])
+                elif len(channel_elements) > 1:
+                    topics[match_topic.group(1)].append(channel_elements)
+            log.debug(f"Found {len(list(topics.keys()))} topics in {self.main_nf}")
+            self.topics = topics

nf_core/module-template/main.nf

@@ -65,7 +65,12 @@ process {{ component_name_underscore|upper }} {
     {{ 'tuple val(meta), path("*")' if has_meta else 'path "*"' }}, emit: output
     {%- endif %}
     {%- endif %}
-    path "versions.yml"           , emit: versions
+    {% if not_empty_template -%}
+    // TODO nf-core: Update the command here to obtain the version number of the software used in this module
+    // TODO nf-core: If multiple software packages are used in this module, all MUST be added here
+    //               by copying the line below and replacing the current tool with the extra tool(s)
+    {%- endif %}
+    tuple val("${task.process}"), val('{{ component }}'), eval("{{ component }} --version"), topic: versions, emit: versions_{{ component }}
 
     when:
     task.ext.when == null || task.ext.when
@@ -111,11 +116,6 @@ process {{ component_name_underscore|upper }} {
         $bam
         {%- endif %}
     {%- endif %}
-
-    cat <<-END_VERSIONS > versions.yml
-    "${task.process}":
-        {{ component }}: \$({{ component }} --version)
-    END_VERSIONS
     """
 
     stub:
@@ -146,10 +146,5 @@ process {{ component_name_underscore|upper }} {
     touch ${prefix}.bam
     {%- endif %}
     {%- endif %}
-
-    cat <<-END_VERSIONS > versions.yml
-    "${task.process}":
-        {{ component }}: \$({{ component }} --version)
-    END_VERSIONS
     """
 }

nf_core/module-template/meta.yml

@@ -46,13 +46,28 @@ output:
             - edam: "http://edamontology.org/format_2572" # BAM
             - edam: "http://edamontology.org/format_2573" # CRAM
             - edam: "http://edamontology.org/format_3462" # SAM
+  versions_{{ component }}:
+    - - "${task.process}":
+          type: string
+          description: The name of the process
+      - "{{ component }}":
+          type: string
+          description: The name of the tool
+      - "{{ component }} --version":
+          type: string
+          description: The version of the tool
+
+topics:
   versions:
-    - "versions.yml":
-        type: file
-        description: File containing software versions
-        pattern: "versions.yml"
-        ontologies:
-          - edam: "http://edamontology.org/format_3750" # YAML
+    - - process:
+          type: string
+          description: The process the versions were collected from
+      - tool:
+          type: string
+          description: The tool name the version was collected for
+      - version:
+          type: string
+          description: The version of the tool
 
 authors:
   - "{{ author }}"

nf_core/modules/lint/__init__.py

@@ -234,6 +234,7 @@ def lint_module(
         if local:
             mod.get_inputs_from_main_nf()
             mod.get_outputs_from_main_nf()
+            mod.get_topics_from_main_nf()
             # Update meta.yml file if requested
             if self.fix and mod.meta_yml is not None:
                 self.update_meta_yml_file(mod)
@@ -260,6 +261,7 @@ def lint_module(
         else:
             mod.get_inputs_from_main_nf()
             mod.get_outputs_from_main_nf()
+            mod.get_topics_from_main_nf()
             # Update meta.yml file if requested
             if self.fix:
                 self.update_meta_yml_file(mod)

nf_core/modules/lint/main_nf.py

@@ -36,14 +36,15 @@ def main_nf(
     * The module has a process label and it is among
       the standard ones.
     * If a ``meta`` map is defined as one of the modules
-      inputs it should be defined as one of the outputs,
+      inputs it should be defined as one of the emits,
       and be correctly configured in the ``saveAs`` function.
     * The module script section should contain definitions
       of ``software`` and ``prefix``
     """
 
     inputs: list[str] = []
-    outputs: list[str] = []
+    emits: list[str] = []
+    topics: list[str] = []
 
     # Check if we have a patch file affecting the 'main.nf' file
     # otherwise read the lines directly from the module
@@ -132,8 +133,9 @@ def main_nf(
                 line = joint_tuple
             inputs.extend(_parse_input(module, line))
         if state == "output" and not _is_empty(line):
-            outputs += _parse_output(module, line)
-            outputs = list(set(outputs))  # remove duplicate 'meta's
+            emits += _parse_output_emits(module, line)
+            emits = list(set(emits))  # remove duplicate 'meta's
+            topics += _parse_output_topics(module, line)
         if state == "when" and not _is_empty(line):
             when_lines.append(line)
         if state == "script" and not _is_empty(line):
@@ -144,7 +146,7 @@ def main_nf(
             exec_lines.append(line)
 
     # Check that we have required sections
-    if not len(outputs):
+    if not len(emits):
         module.failed.append(("main_nf", "main_nf_script_outputs", "No process 'output' block found", module.main_nf))
     else:
         module.passed.append(("main_nf", "main_nf_script_outputs", "Process 'output' block found", module.main_nf))
@@ -192,8 +194,8 @@ def main_nf(
     if inputs:
         if "meta" in inputs:
             module.has_meta = True
-            if outputs:
-                if "meta" in outputs:
+            if emits:
+                if "meta" in emits:
                     module.passed.append(
                         (
                             "main_nf",
@@ -213,22 +215,43 @@ def main_nf(
                     )
 
     # Check that a software version is emitted
-    if outputs:
-        if "versions" in outputs:
+    if topics:
+        if "versions" in topics:
             module.passed.append(
-                ("main_nf", "main_nf_version_emitted", "Module emits software version", module.main_nf)
+                ("main_nf", "main_nf_version_topic", "Module emits software versions as topic", module.main_nf)
             )
         else:
+            module.warned.append(
+                ("main_nf", "main_nf_version_topic", "Module does not emit software versions as topic", module.main_nf)
+            )
+
+    if emits:
+        topic_versions_amount = sum(1 for t in topics if t == "versions")
+        emit_versions_amount = sum(1 for e in emits if e.startswith("versions"))
+        if topic_versions_amount == emit_versions_amount:
+            module.passed.append(
+                ("main_nf", "main_nf_version_emit", "Module emits each software version", module.main_nf)
+            )
+        elif "versions" in emits:
             module.warned.append(
                 (
                     "main_nf",
-                    "main_nf_version_emitted",
-                    "Module does not emit software version",
+                    "main_nf_version_emit",
+                    "Module emits software versions YAML, please update this to topics output",
+                    module.main_nf,
+                )
+            )
+        else:
+            module.failed.append(
+                (
+                    "main_nf",
+                    "main_nf_version_emit",
+                    "Module does not have an `emit:` and `topic:` for each software version",
                     module.main_nf,
                 )
             )
 
-    return inputs, outputs
+    return inputs, emits
 
 
 def check_script_section(self, lines):
@@ -238,14 +261,6 @@ def check_script_section(self, lines):
     """
     script = "".join(lines)
 
-    # check that process name is used for `versions.yml`
-    if re.search(r"\$\{\s*task\.process\s*\}", script):
-        self.passed.append(("main_nf", "main_nf_version_script", "Process name used for versions.yml", self.main_nf))
-    else:
-        self.warned.append(
-            ("main_nf", "main_nf_version_script", "Process name not used for versions.yml", self.main_nf)
-        )
-
     # check for prefix (only if module has a meta map as input)
     if self.has_meta:
         if re.search(r"\s*prefix\s*=\s*task.ext.prefix", script):
@@ -705,16 +720,43 @@ def _parse_input(self, line_raw):
     return inputs
 
 
-def _parse_output(self, line):
+def _parse_output_emits(self, line: str) -> list[str]:
     output = []
     if "meta" in line:
         output.append("meta")
-    if "emit:" not in line:
-        self.failed.append(("main_nf", "missing_emit", f"Missing emit statement: {line.strip()}", self.main_nf))
+    emit_regex = re.search(r"^.*emit:\s*([^,\s]*)", line)
+    if not emit_regex:
+        self.failed.append(("missing_emit", f"Missing emit statement: {line.strip()}", self.main_nf))
     else:
-        output.append(line.split("emit:")[1].strip())
-        self.passed.append(("main_nf", "missing_emit", f"Emit statement found: {line.strip()}", self.main_nf))
+        output.append(emit_regex.group(1).strip())
+    return output
+
 
+def _parse_output_topics(self, line: str) -> list[str]:
+    output = []
+    if "meta" in line:
+        output.append("meta")
+    topic_regex = re.search(r"^.*topic:\s*([^,\s]*)", line)
+    if topic_regex:
+        topic_name = topic_regex.group(1).strip()
+        output.append(topic_name)
+        if topic_name == "versions":
+            if not re.search(r'tuple\s+val\("\${\s*task\.process\s*}"\),\s*val\(.*\),\s*eval\(.*\)', line):
+                self.failed.append(
+                    (
+                        "wrong_version_output",
+                        'Versions topic output is not correctly formatted, expected `tuple val("${task.process}"), val(\'<tool>\'), eval("<version_command>")`',
+                        self.main_nf,
+                    )
+                )
+            if not re.search(r"emit:\s*versions_[\d\w]+", line):
+                self.failed.append(
+                    (
+                        "wrong_version_emit",
+                        "Version emit should follow the format `versions_<tool_or_package>`, e.g.: `versions_samtools`, `versions_gatk4`",
+                        self.main_nf,
+                    )
+                )
     return output