use feature branch from pynxtools-xps

Author: lukaspie

.github/workflows/plugin_test.yaml

@@ -42,7 +42,7 @@ jobs:
             branch: main
             tests_to_run: tests/.
           - plugin: pynxtools-xps
-            branch: main
+            branch: custom-attr-for-open-enum
             tests_to_run: tests/.
           - plugin: pynxtools-xrd
             branch: main

src/pynxtools/dataconverter/helpers.py

@@ -57,8 +57,9 @@ class ValidationProblem(Enum):
     UnitWithoutDocumentation = auto()
     InvalidUnit = auto()
     InvalidEnum = auto()
-    OpenEnumWithCorrectNewItem = auto()
-    OpenEnumWithIncorrectNewItem = auto()
+    OpenEnumWithCustom = auto()
+    OpenEnumWithCustomFalse = auto()
+    OpenEnumWithMissingCustom = auto()
     MissingRequiredGroup = auto()
     MissingRequiredField = auto()
     MissingRequiredAttribute = auto()
@@ -85,7 +86,6 @@ class ValidationProblem(Enum):
     KeysWithAndWithoutConcept = auto()
     InvalidCompressionStrength = auto()
     CompressionStrengthZero = auto()
-    # DoNotCompressStringsBoolean = auto()
 
 
 class Collector:
@@ -119,22 +119,22 @@ def _log(self, path: str, log_type: ValidationProblem, value: Optional[Any], *ar
 
         elif log_type == ValidationProblem.InvalidEnum:
             logger.warning(
-                f"The value at {path} should be one of the following: {value}."
+                f"The value '{args[0]}' at {path} should be one of the following: {value}."
             )
-        elif log_type == ValidationProblem.OpenEnumWithCorrectNewItem:
+        elif log_type == ValidationProblem.OpenEnumWithCustom:
             logger.info(
                 f"The value '{args[0]}' at {path} does not match with the enumerated items from the open enumeration: {value}."
             )
-        elif log_type == ValidationProblem.OpenEnumWithIncorrectNewItem:
-            actual_value, custom_attr = args
-
-            log_text = f"The value '{actual_value}' at {path} does not match with the enumerated items from the open enumeration: {value}."
-            if custom_attr == "custom_missing":
-                log_text += f" When a different value is used, a boolean 'custom' attribute must be added."
-                logger.warning(log_text)
-            elif custom_attr == "custom_false":
-                log_text += f" When a different value is used, the boolean 'custom' attribute cannot be False."
-                logger.warning(log_text)
+        elif log_type == ValidationProblem.OpenEnumWithCustomFalse:
+            logger.warning(
+                f"The value '{args[0]}' at {path} does not match with the enumerated items from the open enumeration: {value}."
+                "When a different value is used, the boolean 'custom' attribute cannot be False."
+            )
+        elif log_type == ValidationProblem.OpenEnumWithMissingCustom:
+            logger.info(
+                f"The value '{args[0]}' at {path} does not match with the enumerated items from the open enumeration: {value}. "
+                "When a different value is used, a boolean 'custom=True' attribute must be added. It was added here automatically."
+            )
 
         elif log_type == ValidationProblem.MissingRequiredGroup:
             logger.warning(f"The required group, {path}, hasn't been supplied.")
@@ -231,18 +231,6 @@ def _log(self, path: str, log_type: ValidationProblem, value: Optional[Any], *ar
             logger.warning(
                 f"Compression strength for {path} = {value} should be between 0 and 9."
             )
-        # elif log_type == ValidationProblem.DoNotCompressStringsBoolean:
-        #     value = cast(dict, value)
-        #     dtype = type(value["compress"]).__name__
-        #     dtype_map = {
-        #         "str": "string",
-        #         "bool": "boolean",
-        #     }
-        #     dtype_str = dtype_map.get(dtype, dtype)
-
-        #     logger.info(
-        #         f"Compression for {path} = {value} should not be used for {dtype_str} values."
-        #     )
 
     def collect_and_log(
         self,
@@ -264,9 +252,9 @@ def collect_and_log(
         # info messages should not fail validation
         if log_type not in (
             ValidationProblem.UnitWithoutDocumentation,
-            ValidationProblem.OpenEnumWithNewItem,
             ValidationProblem.CompressionStrengthZero,
-            ValidationProblem.OpenEnumWithCorrectNewItem,
+            ValidationProblem.OpenEnumWithCustom,
+            ValidationProblem.OpenEnumWithMissingCustom,
         ):
             self.data.add(path + str(log_type) + str(value))
 
@@ -777,10 +765,8 @@ def convert_int_to_float(value):
 def is_valid_data_field(value: Any, nxdl_type: str, path: str) -> Any:
     """Checks whether a given value is valid according to the type defined in the NXDL."""
 
-    def validate_data_value(
-        value: Any, nxdl_type: str, nxdl_enum: list, nxdl_enum_open: bool, path: str
-    ) -> Any:
-        """Validate and possibly convert a primitive value according to NXDL type/enum rules."""
+    def validate_data_value(value: Any, nxdl_type: str, path: str) -> Any:
+        """Validate and possibly convert a primitive value according to NXDL type rules."""
         accepted_types = NEXUS_TO_PYTHON_DATA_TYPES[nxdl_type]
         original_value = value
 
@@ -811,16 +797,6 @@ def validate_data_value(
                     path, ValidationProblem.InvalidDatetime, value
                 )
 
-        if nxdl_enum is not None and value not in nxdl_enum:
-            if nxdl_enum_open:
-                collector.collect_and_log(
-                    path, ValidationProblem.OpenEnumWithNewItem, nxdl_enum
-                )
-            else:
-                collector.collect_and_log(
-                    path, ValidationProblem.InvalidEnum, nxdl_enum
-                )
-
         return value
 
     if isinstance(value, dict) and set(value.keys()) == {"compress", "strength"}:
@@ -836,24 +812,14 @@ def validate_data_value(
                     path, ValidationProblem.InvalidCompressionStrength, value
                 )
             # In this case, we remove the compression.
-            return validate_data_value(
-                value["compress"], nxdl_type, nxdl_enum, nxdl_enum_open, path
-            )
-
-        # TODO: Do we need to issue a warning if string/bool compression is used
-        # # elif isinstance(compressed_value, (str, bool)):
-        #     collector.collect_and_log(
-        #         path, ValidationProblem.DoNotCompressStringsBoolean, value
-        #     )
+            return validate_data_value(value["compress"], nxdl_type, path)
 
         # Apply standard validation to compressed value
-        value["compress"] = validate_data_value(
-            compressed_value, nxdl_type, nxdl_enum, nxdl_enum_open, path
-        )
+        value["compress"] = validate_data_value(compressed_value, nxdl_type, path)
 
         return value
 
-    return validate_data_value(value, nxdl_type, nxdl_enum, nxdl_enum_open, path)
+    return validate_data_value(value, nxdl_type, path)
 
 
 @cache

src/pynxtools/dataconverter/validation.py

@@ -1241,7 +1241,11 @@ def is_valid_enum(
         path: str,
         mapping: MutableMapping,
     ):
-        # Check enumeration
+        """Check enumeration."""
+
+        if isinstance(value, dict) and set(value.keys()) == {"compress", "strength"}:
+            value = value["compress"]
+
         if nxdl_enum is not None and value not in nxdl_enum:
             if nxdl_enum_open:
                 if path.split("/")[-1].startswith("@"):
@@ -1256,23 +1260,29 @@ def is_valid_enum(
                 if custom_attr is True:
                     collector.collect_and_log(
                         path,
-                        ValidationProblem.OpenEnumWithCorrectNewItem,
+                        ValidationProblem.OpenEnumWithCustom,
                         nxdl_enum,
                         value,
                     )
-                else:
+                elif custom_attr is False:
+                    collector.collect_and_log(
+                        path,
+                        ValidationProblem.OpenEnumWithCustomFalse,
+                        nxdl_enum,
+                        value,
+                    )
+
+                elif custom_attr is None:
+                    mapping[path] = True
                     collector.collect_and_log(
                         path,
-                        ValidationProblem.OpenEnumWithIncorrectNewItem,
+                        ValidationProblem.OpenEnumWithMissingCustom,
                         nxdl_enum,
                         value,
-                        "custom_false" if custom_attr is False else "custom_missing",
                     )
             else:
                 collector.collect_and_log(
-                    path,
-                    ValidationProblem.InvalidEnum,
-                    nxdl_enum,
+                    path, ValidationProblem.InvalidEnum, nxdl_enum, value
                 )
 
     def check_reserved_suffix(key: str, mapping: MutableMapping[str, Any]) -> bool:

tests/dataconverter/test_validation.py

@@ -856,6 +856,32 @@ def listify_template(data_dict: Template):
             ],
             id="open-enum-with-new-item",
         ),
+        pytest.param(
+            alter_dict(
+                alter_dict(
+                    alter_dict(
+                        alter_dict(
+                            TEMPLATE,
+                            "/ENTRY[my_entry]/NXODD_name[nxodd_name]/type2",
+                            {"compress": "a very different type", "strength": 2},
+                        ),
+                        "/ENTRY[my_entry]/NXODD_name[nxodd_name]/type2/@custom",
+                        True,
+                    ),
+                    "/ENTRY[my_entry]/NXODD_name[nxodd_name]/type2/@attribute_with_open_enum",
+                    {"compress": "3rd option", "strength": 4},
+                ),
+                "/ENTRY[my_entry]/NXODD_name[nxodd_name]/type2/@attribute_with_open_enum_custom",
+                True,
+            ),
+            [
+                "The value 'a very different type' at /ENTRY[my_entry]/NXODD_name[nxodd_name]/type2 does not match "
+                "with the enumerated items from the open enumeration: ['1st type open', '2nd type open'].",
+                "The value '3rd option' at /ENTRY[my_entry]/NXODD_name[nxodd_name]/type2/@attribute_with_open_enum "
+                "does not match with the enumerated items from the open enumeration: ['1st option', '2nd option'].",
+            ],
+            id="open-enum-with-new-item-compressed",
+        ),
         pytest.param(
             alter_dict(
                 alter_dict(
@@ -1879,12 +1905,14 @@ def format_error_message(msg: str) -> str:
             "field-with-illegal-unit",
             "baseclass-field-with-illegal-unit",
             "open-enum-with-new-item",
+            "open-enum-with-new-item-compressed",
             "baseclass-open-enum-with-new-item",
             "appdef-compressed-strength-0",
         ):
             with caplog.at_level(logging.INFO):
                 assert validate_dict_against("NXtest", data_dict)
-                assert error_messages[0] in caplog.text
+                for expected_message, rec in zip(error_messages, caplog.records):
+                    assert expected_message == format_error_message(rec.message)
         else:
             with caplog.at_level(logging.WARNING):
                 assert not validate_dict_against("NXtest", data_dict)