Write definition and version directly independently from the reader (#312)

* Emit a sensible error message if the nxdl does not exist

* Fix test for non existing nxdl

* Write definition and version automatically

* Don't overwrite version

* Move get entry names to template

Author: domna

pynxtools/dataconverter/convert.py

@@ -207,6 +207,9 @@ def transfer_data_into_template(
     data = data_reader().read(  # type: ignore[operator]
         template=Template(template), file_paths=input_file, **kwargs
     )
+    entry_names = data.get_all_entry_names()
+    for entry_name in entry_names:
+        helpers.write_nexus_def_to_entry(data, entry_name, nxdl_name)
     if not skip_verify:
         helpers.validate_data_dict(template, data, nxdl_root)
     return data
@@ -423,15 +426,20 @@ def convert_cli(
             "The --input-file option is deprecated. Please use the positional arguments instead."
         )
 
-    convert(
-        tuple(file_list) + input_file,
-        reader,
-        nxdl,
-        output,
-        fair,
-        undocumented,
-        skip_verify,
-    )
+    try:
+        convert(
+            tuple(file_list) + input_file,
+            reader,
+            nxdl,
+            output,
+            fair,
+            undocumented,
+            skip_verify,
+        )
+    except FileNotFoundError as exc:
+        raise click.BadParameter(
+            f"{nxdl} is not a valid application definition", param_hint="--nxdl"
+        ) from exc
 
 
 @main_cli.command()

pynxtools/dataconverter/helpers.py

@@ -686,7 +686,7 @@ def add_default_root_attributes(data, filename):
     def update_and_warn(key: str, value: str):
         if key in data and data[key] != value:
             logger.warning(
-                f"The NXroot entry '{key}' (value: {data[key]}) should not be populated by "
+                f"The NXroot entry '{key}' (value: {data[key]}) should not be changed by "
                 f"the reader. This is overwritten by the actually used value '{value}'"
             )
         data[key] = value
@@ -705,6 +705,34 @@ def update_and_warn(key: str, value: str):
     update_and_warn("/@h5py_version", h5py.__version__)
 
 
+def write_nexus_def_to_entry(data, entry_name: str, nxdl_def: str):
+    """
+    Writes the used nexus definition and version to /ENTRY/definition
+    """
+
+    def update_and_warn(key: str, value: str, overwrite=False):
+        if key in data and data[key] is not None and data[key] != value:
+            report = (
+                f"This is overwritten by the actually used value '{value}'"
+                if overwrite
+                else f"The provided version '{value}' is kept. We assume you know what you are doing."
+            )
+            logger.log(
+                logging.WARNING if overwrite else logging.INFO,
+                f"The entry '{key}' (value: {data[key]}) should not be changed by "
+                f"the reader. {report}",
+            )
+        if overwrite or data.get(key) is None:
+            data[key] = value
+
+    update_and_warn(f"/ENTRY[{entry_name}]/definition", nxdl_def, overwrite=True)
+    update_and_warn(
+        f"/ENTRY[{entry_name}]/definition/@version",
+        get_nexus_version(),
+        overwrite=False,
+    )
+
+
 def extract_atom_types(formula, mode="hill"):
     """Extract atom types form chemical formula."""
     atom_types: set = set()

pynxtools/dataconverter/template.py

@@ -19,6 +19,8 @@
 
 import copy
 import json
+import re
+from typing import Set
 
 from pynxtools.dataconverter import helpers
 
@@ -197,6 +199,20 @@ def rename_entry(self, old_name: str, new_name: str, deepcopy=True):
                     internal_dict[f"/ENTRY[{new_name}]{rest_of_path}"] = value
                     del internal_dict[key]
 
+    def get_all_entry_names(self) -> Set[str]:
+        """
+        Get all entry names in the template.
+
+        Returns:
+            Set[str]: A set of entry names.
+        """
+        entry_names = set()
+        for key in self:
+            entry_name_match = re.search(r"\/ENTRY\[([a-zA-Z0-9_\.]+)\]", key)
+            if entry_name_match is not None:
+                entry_names.add(entry_name_match.group(1))
+        return entry_names
+
     def update(self, template):
         """Merges second template to original
         or updates values from a dictionary if the type of :code:`template` is dict"""

tests/dataconverter/test_convert.py

@@ -21,6 +21,7 @@
 import os
 from pathlib import Path
 
+import click
 import h5py
 import pytest
 from click.testing import CliRunner
@@ -83,7 +84,11 @@ def test_find_nxdl(cli_inputs):
     runner = CliRunner()
     result = runner.invoke(dataconverter.convert_cli, cli_inputs)
     if "NXdoesnotexist" in cli_inputs:
-        assert isinstance(result.exception, FileNotFoundError)
+        assert result.exit_code == 2
+        assert result.output.endswith(
+            "Error: Invalid value for --nxdl: "
+            "NXdoesnotexist is not a valid application definition\n"
+        )
     else:
         assert isinstance(result.exception, Exception)
         assert "The chosen NXDL isn't supported by the selected reader." in str(

tests/dataconverter/test_helpers.py

@@ -483,6 +483,8 @@ def test_writing_of_root_attributes(caplog):
     filename = "my_nexus_file.nxs"
     with caplog.at_level(logging.WARNING):
         helpers.add_default_root_attributes(template, filename)
+        helpers.write_nexus_def_to_entry(template, "entry", "NXtest")
+        helpers.write_nexus_def_to_entry(template, "entry1", "NXtest")
 
     assert "" == caplog.text
 
@@ -497,6 +499,10 @@ def test_writing_of_root_attributes(caplog):
     assert "/@NeXus_version" in keys_added
     assert "/@HDF5_version" in keys_added
     assert "/@h5py_version" in keys_added
+    assert "/ENTRY[entry]/definition" in keys_added
+    assert "/ENTRY[entry]/definition/@version" in keys_added
+    assert "/ENTRY[entry1]/definition" in keys_added
+    assert "/ENTRY[entry1]/definition/@version" in keys_added
 
 
 def test_warning_on_root_attribute_overwrite(caplog):
@@ -510,10 +516,26 @@ def test_warning_on_root_attribute_overwrite(caplog):
     with caplog.at_level(logging.WARNING):
         helpers.add_default_root_attributes(template, filname)
     error_text = (
-        "The NXroot entry '/@NX_class' (value: NXwrong) should not be populated by the reader. "
+        "The NXroot entry '/@NX_class' (value: NXwrong) should not be changed by the reader. "
         "This is overwritten by the actually used value 'NXroot'"
     )
     assert error_text in caplog.text
 
     assert "/@NX_class" in template.keys()
     assert template["/@NX_class"] == "NXroot"
+
+
+def test_warning_on_definition_changed_by_reader(caplog):
+    template = Template()
+    template["/ENTRY[entry]/definition"] = "NXwrong"
+    with caplog.at_level(logging.WARNING):
+        helpers.write_nexus_def_to_entry(template, "entry", "NXtest")
+
+    error_text = (
+        "The entry '/ENTRY[entry]/definition' (value: NXwrong) should not be changed by the reader. "
+        "This is overwritten by the actually used value 'NXtest'"
+    )
+    assert error_text in caplog.text
+
+    assert "/ENTRY[entry]/definition" in template.keys()
+    assert template["/ENTRY[entry]/definition"] == "NXtest"