support custom attribute in validate_nexus

Author: lukaspie

src/pynxtools/dataconverter/helpers.py

@@ -21,7 +21,7 @@
 import logging
 import os
 import re
-from collections.abc import Mapping, Sequence
+from collections.abc import Mapping, MutableMapping, Sequence
 from datetime import datetime, timezone
 from enum import Enum, auto
 from functools import cache, lru_cache
@@ -154,7 +154,7 @@ def _log(self, path: str, log_type: ValidationProblem, value: Optional[Any], *ar
 
         elif log_type == ValidationProblem.InvalidEnum:
             logger.warning(
-                f"The value {args[0]} at {path} should be one of the following: {value}."
+                f"The value '{args[0]}' at {path} should be one of the following: {value}."
             )
         elif log_type == ValidationProblem.OpenEnumWithCustom:
             logger.info(
@@ -166,11 +166,13 @@ def _log(self, path: str, log_type: ValidationProblem, value: Optional[Any], *ar
                 "When a different value is used, the boolean 'custom' attribute cannot be False."
             )
         elif log_type == ValidationProblem.OpenEnumWithMissingCustom:
-            logger.info(
+            log_text = (
                 f"The value '{args[0]}' at {path} does not match with the enumerated items from the open enumeration: {value}. "
-                "When a different value is used, a boolean 'custom=True' attribute must be added. It was added here automatically."
+                "When a different value is used, a boolean 'custom=True' attribute must be added."
             )
-
+            if args[1] is True:
+                log_text += " It was added here automatically."
+            logger.info(log_text)
         elif log_type == ValidationProblem.MissingRequiredGroup:
             logger.warning(f"The required group {path} hasn't been supplied.")
         elif log_type == ValidationProblem.MissingRequiredField:
@@ -878,6 +880,81 @@ def validate_data_value(value: Any, nxdl_type: str, path: str) -> Any:
     return validate_data_value(value, nxdl_type, path)
 
 
+def get_custom_attr_path(path: str) -> str:
+    if path.split("/")[-1].startswith("@"):
+        attr_name = path.split("/")[-1][1:]  # remove "@"
+        return f"{path}_custom"
+    return f"{path}/@custom"
+
+
+def is_valid_enum(
+    value: Any,
+    nxdl_enum: list,
+    nxdl_enum_open: bool,
+    path: str,
+    mapping: MutableMapping,
+):
+    """Check enumeration."""
+
+    if isinstance(value, dict) and set(value.keys()) == {"compress", "strength"}:
+        value = value["compress"]
+
+    if nxdl_enum is not None:
+        if (
+            isinstance(value, np.ndarray)
+            and isinstance(nxdl_enum, list)
+            and isinstance(nxdl_enum[0], list)
+        ):
+            enum_value = list(value)
+        else:
+            enum_value = value
+
+        if enum_value not in nxdl_enum:
+            if nxdl_enum_open:
+                custom_path = get_custom_attr_path(path)
+
+                if isinstance(mapping, h5py.Group):
+                    parent_path, attr_name = custom_path.rsplit("@", 1)
+                    custom_attr = mapping.get(parent_path).attrs.get(attr_name)
+                    custom_added_auto = False
+                else:
+                    custom_attr = mapping.get(custom_path)
+                    custom_added_auto = True
+
+                if custom_attr == True:  # noqa: E712
+                    collector.collect_and_log(
+                        path,
+                        ValidationProblem.OpenEnumWithCustom,
+                        nxdl_enum,
+                        value,
+                    )
+                elif custom_attr == False:  # noqa: E712
+                    collector.collect_and_log(
+                        path,
+                        ValidationProblem.OpenEnumWithCustomFalse,
+                        nxdl_enum,
+                        value,
+                    )
+
+                elif custom_attr is None:
+                    try:
+                        mapping[custom_path] = True
+                    except ValueError:
+                        # we are in the HDF5 validation, cannot set custom attribute.
+                        pass
+                    collector.collect_and_log(
+                        path,
+                        ValidationProblem.OpenEnumWithMissingCustom,
+                        nxdl_enum,
+                        value,
+                        custom_added_auto,
+                    )
+            else:
+                collector.collect_and_log(
+                    path, ValidationProblem.InvalidEnum, nxdl_enum, value
+                )
+
+
 def split_class_and_name_of(name: str) -> tuple[Optional[str], str]:
     """
     Return the class and the name of a data dict entry of the form

src/pynxtools/dataconverter/validation.py

@@ -41,7 +41,9 @@
     clean_str_attr,
     collector,
     convert_nexus_to_caps,
+    get_custom_attr_path,
     is_valid_data_field,
+    is_valid_enum,
     split_class_and_name_of,
 )
 from pynxtools.dataconverter.nexus_tree import (
@@ -644,9 +646,14 @@ def handle_field(
         is_valid_data_field(
             clean_str_attr(dataset[()]),
             node.dtype,
+            full_path,
+        )
+        is_valid_enum(
+            clean_str_attr(dataset[()]),
             node.items,
             node.open_enum,
             full_path,
+            data,
         )
 
         units = dataset.attrs.get("units")
@@ -695,7 +702,12 @@ def handle_attributes(
         for attr_name in attrs:
             full_path = f"{entry_name}/{path}/@{attr_name}"
 
-            if attr_name in ("NX_class", "units", "target"):
+            if attr_name in (
+                "NX_class",
+                "units",
+                "target",
+                "custom",
+            ) or attr_name.endswith("_custom"):
                 # Ignore special attrs
                 continue
 
@@ -734,9 +746,15 @@ def handle_attributes(
             is_valid_data_field(
                 attr_data,
                 node.dtype,
+                full_path,
+            )
+
+            is_valid_enum(
+                attr_data,
                 node.items,
                 node.open_enum,
                 full_path,
+                data,
             )
 
     def validate(path: str, h5_obj: Union[h5py.Group, h5py.Dataset]):
@@ -1351,6 +1369,7 @@ def handle_field(node: NexusNode, keys: Mapping[str, Any], prev_path: str):
                 variant_path,
                 mapping,
             )
+            remove_from_not_visited(get_custom_attr_path(variant_path))
 
             check_reserved_suffix(variant_path, keys)
             check_reserved_prefix(variant_path, get_definition(variant_path), "field")
@@ -1424,6 +1443,7 @@ def handle_attribute(node: NexusNode, keys: Mapping[str, Any], prev_path: str):
                 variant_path,
                 mapping,
             )
+            remove_from_not_visited(get_custom_attr_path(variant_path))
             check_reserved_prefix(
                 variant_path, get_definition(variant_path), "attribute"
             )
@@ -1662,6 +1682,7 @@ def is_documented(key: str, tree: NexusNode) -> bool:
                     key,
                     mapping,
                 )
+                remove_from_not_visited(get_custom_attr_path(key))
 
             return True
 
@@ -1686,6 +1707,7 @@ def is_documented(key: str, tree: NexusNode) -> bool:
             key,
             mapping,
         )
+        remove_from_not_visited(get_custom_attr_path(key))
 
         # Check main field exists for units
         if (