search for custom attribute with open enum

Author: lukaspie

src/pynxtools/data/NXtest.nxdl.xml

@@ -99,6 +99,12 @@
                     <item value="1st type open"/>
                     <item value="2nd type open"/>
                 </enumeration>
+                <attribute name="attribute_with_open_enum" optional="true">
+                    <enumeration open="true">
+                        <item value="1st option"/>
+                        <item value="2nd option"/>
+                    </enumeration>
+            </attribute>
             </field>
             <attribute name="group_attribute">
             </attribute>

src/pynxtools/dataconverter/helpers.py

@@ -57,7 +57,8 @@ class ValidationProblem(Enum):
     UnitWithoutDocumentation = auto()
     InvalidUnit = auto()
     InvalidEnum = auto()
-    OpenEnumWithNewItem = auto()
+    OpenEnumWithCorrectNewItem = auto()
+    OpenEnumWithIncorrectNewItem = auto()
     MissingRequiredGroup = auto()
     MissingRequiredField = auto()
     MissingRequiredAttribute = auto()
@@ -120,10 +121,21 @@ def _log(self, path: str, log_type: ValidationProblem, value: Optional[Any], *ar
             logger.warning(
                 f"The value at {path} should be one of the following: {value}."
             )
-        elif log_type == ValidationProblem.OpenEnumWithNewItem:
+        elif log_type == ValidationProblem.OpenEnumWithCorrectNewItem:
             logger.info(
-                f"The value at {path} does not match with the enumerated items from the open enumeration: {value}."
+                f"The value '{args[0]}' at {path} does not match with the enumerated items from the open enumeration: {value}."
             )
+        elif log_type == ValidationProblem.OpenEnumWithIncorrectNewItem:
+            actual_value, custom_attr = args
+
+            log_text = f"The value '{actual_value}' at {path} does not match with the enumerated items from the open enumeration: {value}."
+            if custom_attr == "custom_missing":
+                log_text += f" When a different value is used, a boolean 'custom' attribute must be added."
+                logger.warning(log_text)
+            elif custom_attr == "custom_false":
+                log_text += f" When a different value is used, the boolean 'custom' attribute cannot be False."
+                logger.warning(log_text)
+
         elif log_type == ValidationProblem.MissingRequiredGroup:
             logger.warning(f"The required group, {path}, hasn't been supplied.")
         elif log_type == ValidationProblem.MissingRequiredField:
@@ -254,6 +266,7 @@ def collect_and_log(
             ValidationProblem.UnitWithoutDocumentation,
             ValidationProblem.OpenEnumWithNewItem,
             ValidationProblem.CompressionStrengthZero,
+            ValidationProblem.OpenEnumWithCorrectNewItem,
         ):
             self.data.add(path + str(log_type) + str(value))
 
@@ -761,9 +774,7 @@ def convert_int_to_float(value):
         return value
 
 
-def is_valid_data_field(
-    value: Any, nxdl_type: str, nxdl_enum: list, nxdl_enum_open: bool, path: str
-) -> Any:
+def is_valid_data_field(value: Any, nxdl_type: str, path: str) -> Any:
     """Checks whether a given value is valid according to the type defined in the NXDL."""
 
     def validate_data_value(

src/pynxtools/dataconverter/validation.py

@@ -593,7 +593,7 @@ def _follow_link(
                     resolved_keys[key] = current_keys
         return resolved_keys
 
-    def handle_field(node: NexusNode, keys: Mapping[str, Any], prev_path: str):
+    def handle_field(node: NexusEntity, keys: Mapping[str, Any], prev_path: str):
         full_path = remove_from_not_visited(f"{prev_path}/{node.name}")
         variants = get_variations_of(node, keys)
         if (
@@ -651,13 +651,18 @@ def handle_field(node: NexusNode, keys: Mapping[str, Any], prev_path: str):
             mapping[variant_path] = is_valid_data_field(
                 mapping[variant_path],
                 node.dtype,
+                variant_path,
+            )
+            is_valid_enum(
+                mapping[variant_path],
                 node.items,
                 node.open_enum,
                 variant_path,
+                mapping,
             )
 
-            _ = check_reserved_suffix(variant_path, mapping)
-            _ = check_reserved_prefix(variant_path, mapping, "field")
+            check_reserved_suffix(variant_path, mapping)
+            check_reserved_prefix(variant_path, mapping, "field")
 
             # Check unit category
             if node.unit is not None:
@@ -695,7 +700,7 @@ def handle_field(node: NexusNode, keys: Mapping[str, Any], prev_path: str):
                 prev_path=variant_path,
             )
 
-    def handle_attribute(node: NexusNode, keys: Mapping[str, Any], prev_path: str):
+    def handle_attribute(node: NexusEntity, keys: Mapping[str, Any], prev_path: str):
         full_path = remove_from_not_visited(f"{prev_path}/@{node.name}")
         variants = get_variations_of(node, keys)
 
@@ -716,11 +721,16 @@ def handle_attribute(node: NexusNode, keys: Mapping[str, Any], prev_path: str):
                     f"{prev_path}/{variant if variant.startswith('@') else f'@{variant}'}"
                 ],
                 node.dtype,
+                variant_path,
+            )
+            is_valid_enum(
+                mapping[variant_path],
                 node.items,
                 node.open_enum,
                 variant_path,
+                mapping,
             )
-            _ = check_reserved_prefix(variant_path, mapping, "attribute")
+            check_reserved_prefix(variant_path, mapping, "attribute")
 
     def handle_choice(node: NexusNode, keys: Mapping[str, Any], prev_path: str):
         global collector
@@ -866,7 +876,16 @@ def is_documented(key: str, tree: NexusNode) -> bool:
                     # keys_to_remove.append(key)
                     # return False
                 resolved_link[key] = is_valid_data_field(
-                    resolved_link[key], node.dtype, node.items, node.open_enum, key
+                    resolved_link[key],
+                    node.dtype,
+                    key,
+                )
+                is_valid_enum(
+                    resolved_link[key],
+                    node.items,
+                    node.open_enum,
+                    key,
+                    mapping,
                 )
 
             return True
@@ -881,7 +900,16 @@ def is_documented(key: str, tree: NexusNode) -> bool:
 
         # Check general validity
         mapping[key] = is_valid_data_field(
-            mapping[key], node.dtype, node.items, node.open_enum, key
+            mapping[key],
+            node.dtype,
+            key,
+        )
+        is_valid_enum(
+            mapping[key],
+            node.items,
+            node.open_enum,
+            key,
+            mapping,
         )
 
         # Check main field exists for units
@@ -1206,6 +1234,47 @@ def startswith_with_variations(
         # default
         return (False, 0)
 
+    def is_valid_enum(
+        value: Any,
+        nxdl_enum: list,
+        nxdl_enum_open: bool,
+        path: str,
+        mapping: MutableMapping,
+    ):
+        # Check enumeration
+        if nxdl_enum is not None and value not in nxdl_enum:
+            if nxdl_enum_open:
+                if path.split("/")[-1].startswith("@"):
+                    attr_name = path.split("/")[-1][1:]  # remove "@"
+                    custom_path = f"{path}_custom"
+                else:
+                    custom_path = f"{path}/@custom"
+
+                custom_attr = mapping.get(custom_path)
+                remove_from_not_visited(custom_path)
+
+                if custom_attr is True:
+                    collector.collect_and_log(
+                        path,
+                        ValidationProblem.OpenEnumWithCorrectNewItem,
+                        nxdl_enum,
+                        value,
+                    )
+                else:
+                    collector.collect_and_log(
+                        path,
+                        ValidationProblem.OpenEnumWithIncorrectNewItem,
+                        nxdl_enum,
+                        value,
+                        "custom_false" if custom_attr is False else "custom_missing",
+                    )
+            else:
+                collector.collect_and_log(
+                    path,
+                    ValidationProblem.InvalidEnum,
+                    nxdl_enum,
+                )
+
     def check_reserved_suffix(key: str, mapping: MutableMapping[str, Any]) -> bool:
         """
         Check if an associated field exists for a key with a reserved suffix.

tests/dataconverter/test_validation.py

@@ -832,16 +832,78 @@ def listify_template(data_dict: Template):
         ),
         pytest.param(
             alter_dict(
-                TEMPLATE,
-                "/ENTRY[my_entry]/NXODD_name[nxodd_name]/type2",
-                "a very different type",
+                alter_dict(
+                    alter_dict(
+                        alter_dict(
+                            TEMPLATE,
+                            "/ENTRY[my_entry]/NXODD_name[nxodd_name]/type2",
+                            "a very different type",
+                        ),
+                        "/ENTRY[my_entry]/NXODD_name[nxodd_name]/type2/@custom",
+                        True,
+                    ),
+                    "/ENTRY[my_entry]/NXODD_name[nxodd_name]/type2/@attribute_with_open_enum",
+                    "3rd option",
+                ),
+                "/ENTRY[my_entry]/NXODD_name[nxodd_name]/type2/@attribute_with_open_enum_custom",
+                True,
             ),
             [
-                "The value at /ENTRY[my_entry]/NXODD_name[nxodd_name]/type2 does not match with the "
-                "enumerated items from the open enumeration: ['1st type open', '2nd type open']."
+                "The value 'a very different type' at /ENTRY[my_entry]/NXODD_name[nxodd_name]/type2 does not match "
+                "with the enumerated items from the open enumeration: ['1st type open', '2nd type open'].",
+                "The value '3rd option' at /ENTRY[my_entry]/NXODD_name[nxodd_name]/type2/@attribute_with_open_enum "
+                "does not match with the enumerated items from the open enumeration: ['1st option', '2nd option'].",
             ],
             id="open-enum-with-new-item",
         ),
+        pytest.param(
+            alter_dict(
+                alter_dict(
+                    alter_dict(
+                        alter_dict(
+                            TEMPLATE,
+                            "/ENTRY[my_entry]/NXODD_name[nxodd_name]/type2",
+                            "a very different type",
+                        ),
+                        "/ENTRY[my_entry]/NXODD_name[nxodd_name]/type2/@custom",
+                        False,
+                    ),
+                    "/ENTRY[my_entry]/NXODD_name[nxodd_name]/type2/@attribute_with_open_enum",
+                    "3rd option",
+                ),
+                "/ENTRY[my_entry]/NXODD_name[nxodd_name]/type2/@attribute_with_open_enum_custom",
+                False,
+            ),
+            [
+                "The value 'a very different type' at /ENTRY[my_entry]/NXODD_name[nxodd_name]/type2 does not match "
+                "with the enumerated items from the open enumeration: ['1st type open', '2nd type open']. "
+                "When a different value is used, the boolean 'custom' attribute cannot be False.",
+                "The value '3rd option' at /ENTRY[my_entry]/NXODD_name[nxodd_name]/type2/@attribute_with_open_enum "
+                "does not match with the enumerated items from the open enumeration: ['1st option', '2nd option']. "
+                "When a different value is used, the boolean 'custom' attribute cannot be False.",
+            ],
+            id="open-enum-with-new-item-custom-false",
+        ),
+        pytest.param(
+            alter_dict(
+                alter_dict(
+                    TEMPLATE,
+                    "/ENTRY[my_entry]/NXODD_name[nxodd_name]/type2",
+                    "a very different type",
+                ),
+                "/ENTRY[my_entry]/NXODD_name[nxodd_name]/type2/@attribute_with_open_enum",
+                "3rd option",
+            ),
+            [
+                "The value 'a very different type' at /ENTRY[my_entry]/NXODD_name[nxodd_name]/type2 does not match "
+                "with the enumerated items from the open enumeration: ['1st type open', '2nd type open']. "
+                "When a different value is used, a boolean 'custom' attribute must be added.",
+                "The value '3rd option' at /ENTRY[my_entry]/NXODD_name[nxodd_name]/type2/@attribute_with_open_enum "
+                "does not match with the enumerated items from the open enumeration: ['1st option', '2nd option']. "
+                "When a different value is used, a boolean 'custom' attribute must be added.",
+            ],
+            id="open-enum-with-new-item-custom-missing",
+        ),
         pytest.param(
             set_to_none_in_dict(
                 TEMPLATE, "/ENTRY[my_entry]/optional_parent/required_child", "required"