From 0a432843e1cd18327e22cfa19470d925c5bc4231 Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Mon, 4 Aug 2025 10:57:39 +0200 Subject: [PATCH 1/8] implement more rigoruous checkings of compressed payloads --- src/pynxtools/dataconverter/helpers.py | 139 ++++++++++++++++--------- tests/dataconverter/test_validation.py | 89 +++++++++++++--- 2 files changed, 167 insertions(+), 61 deletions(-) diff --git a/src/pynxtools/dataconverter/helpers.py b/src/pynxtools/dataconverter/helpers.py index c186ff6af..3a17f6938 100644 --- a/src/pynxtools/dataconverter/helpers.py +++ b/src/pynxtools/dataconverter/helpers.py @@ -46,6 +46,12 @@ logger = logging.getLogger("pynxtools") +ISO8601 = re.compile( + r"^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2}(?:" + r"\.\d*)?)(((?!-00:00)(\+|-)(\d{2}):(\d{2})|Z){1})$" +) + + class ValidationProblem(Enum): DifferentVariadicNodesWithTheSameName = auto() UnitWithoutDocumentation = auto() @@ -76,6 +82,9 @@ class ValidationProblem(Enum): ReservedPrefixInWrongContext = auto() InvalidNexusTypeForNamedConcept = auto() KeysWithAndWithoutConcept = auto() + InvalidCompressionStrength = auto() + DoNotCompressEnum = auto() + DoNotCompressStringsBoolean = auto() class Collector: @@ -200,6 +209,25 @@ def _log(self, path: str, log_type: ValidationProblem, value: Optional[Any], *ar logger.warning( f"The key '{path}' uses the valid concept name '{args[0]}', but there is another valid key {value} that uses the non-variadic name of the node.'" ) + elif log_type == ValidationProblem.InvalidCompressionStrength: + logger.warning( + f"Compression strength for {path} = {value} should be between 0 and 9." + ) + elif log_type == ValidationProblem.DoNotCompressEnum: + logger.warning( + f"Compression for {path} = {value} should not be used for enumerated concepts." + ) + elif log_type == ValidationProblem.DoNotCompressStringsBoolean: + dtype = type(value["compress"]).__name__ + dtype_map = { + "str": "string", + "bool": "boolean", + } + dtype_str = dtype_map.get(dtype, "") + + logger.warning( + f"Compression for {path} = {value} should not be used for {dtype_str} values." + ) def collect_and_log( self, @@ -723,6 +751,8 @@ def convert_int_to_float(value): return {convert_int_to_float(v) for v in value} elif isinstance(value, np.ndarray) and np.issubdtype(value.dtype, np.integer): return value.astype(float) + elif isinstance(value, np.generic) and np.issubdtype(type(value), np.integer): + return float(value) else: return value @@ -730,71 +760,82 @@ def convert_int_to_float(value): def is_valid_data_field( value: Any, nxdl_type: str, nxdl_enum: list, nxdl_enum_open: bool, path: str ) -> Any: - # todo: Check this function and write test for it. It seems the function is not - # working as expected. - """Checks whether a given value is valid according to the type defined in the NXDL. - - This function only tries to convert boolean value in str format (e.g. "true" ) to - python Boolean (True). In case, it fails to convert, it raises an Exception. + """Checks whether a given value is valid according to the type defined in the NXDL.""" + + def validate_data_value( + value: Any, nxdl_type: str, nxdl_enum: list, nxdl_enum_open: bool, path: str + ) -> Any: + """Validate and possibly convert a primitive value according to NXDL type/enum rules.""" + accepted_types = NEXUS_TO_PYTHON_DATA_TYPES[nxdl_type] + original_value = value + + # Do not count other dicts as they represent a link value + if not isinstance(value, dict): + # Attempt type conversion + if accepted_types[0] is bool and isinstance(value, str): + try: + value = convert_str_to_bool_safe(value) + except (ValueError, TypeError): + value = original_value + elif accepted_types[0] is float: + value = convert_int_to_float(value) - Return: - value: the possibly converted data value - """ + if not is_valid_data_type(value, accepted_types): + collector.collect_and_log( + path, ValidationProblem.InvalidType, accepted_types, nxdl_type + ) - accepted_types = NEXUS_TO_PYTHON_DATA_TYPES[nxdl_type] + # Type-specific validation + if nxdl_type == "NX_POSINT" and not is_positive_int(value): + collector.collect_and_log(path, ValidationProblem.IsNotPosInt, value) - if isinstance(value, dict) and set(value.keys()) == {"compress", "strength"}: - value = value["compress"] + if nxdl_type in ("ISO8601", "NX_DATE_TIME"): + results = ISO8601.search(value) + if results is None: + collector.collect_and_log( + path, ValidationProblem.InvalidDatetime, value + ) - # Do not count other dicts as they represent a link value - if not isinstance(value, dict) and not is_valid_data_type(value, accepted_types): - # try to convert string to bool - if accepted_types[0] is bool and isinstance(value, str): - try: - value = convert_str_to_bool_safe(value) - except (ValueError, TypeError): + if nxdl_enum is not None and value not in nxdl_enum: + if nxdl_enum_open: collector.collect_and_log( - path, ValidationProblem.InvalidType, accepted_types, nxdl_type + path, ValidationProblem.OpenEnumWithNewItem, nxdl_enum ) - elif accepted_types[0] is float: - value = convert_int_to_float(value) - if not is_valid_data_type(value, accepted_types): + else: collector.collect_and_log( - path, ValidationProblem.InvalidType, accepted_types, nxdl_type + path, ValidationProblem.InvalidEnum, nxdl_enum ) - else: - collector.collect_and_log( - path, ValidationProblem.InvalidType, accepted_types, nxdl_type - ) - if nxdl_type == "NX_POSINT" and not is_positive_int(value): - collector.collect_and_log(path, ValidationProblem.IsNotPosInt, value) + return value - if nxdl_type in ("ISO8601", "NX_DATE_TIME"): - iso8601 = re.compile( - r"^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2}(?:" - r"\.\d*)?)(((?!-00:00)(\+|-)(\d{2}):(\d{2})|Z){1})$" - ) - results = iso8601.search(value) - if results is None: - collector.collect_and_log(path, ValidationProblem.InvalidDatetime, value) + if isinstance(value, dict) and set(value.keys()) == {"compress", "strength"}: + compressed_value = value["compress"] - # Check enumeration - if nxdl_enum is not None and value not in nxdl_enum: - if nxdl_enum_open: + if not (0 <= value["strength"] <= value["strength"] <= 9): collector.collect_and_log( - path, - ValidationProblem.OpenEnumWithNewItem, - nxdl_enum, + path, ValidationProblem.InvalidCompressionStrength, value ) - else: + # In this case, we remove the compression. + return validate_data_value( + value["compress"], nxdl_type, nxdl_enum, nxdl_enum_open, path + ) + + if nxdl_enum is not None: + collector.collect_and_log(path, ValidationProblem.DoNotCompressEnum, value) + + elif isinstance(compressed_value, (str, bool)): collector.collect_and_log( - path, - ValidationProblem.InvalidEnum, - nxdl_enum, + path, ValidationProblem.DoNotCompressStringsBoolean, value ) - return value + # Apply standard validation to compressed value + value["compress"] = validate_data_value( + compressed_value, nxdl_type, nxdl_enum, nxdl_enum_open, path + ) + + return value + + return validate_data_value(value, nxdl_type, nxdl_enum, nxdl_enum_open, path) @cache diff --git a/tests/dataconverter/test_validation.py b/tests/dataconverter/test_validation.py index 9e4f20d43..d2e3addd5 100644 --- a/tests/dataconverter/test_validation.py +++ b/tests/dataconverter/test_validation.py @@ -17,6 +17,7 @@ # limitations under the License. # import logging +import random from typing import Optional import numpy as np @@ -52,6 +53,33 @@ def set_whole_group_to_none( return internal_dict +def compress_paths_in_dict(data_dict: Template, paths=list[str]): + types = { + "int": np.int64, + "float": np.float32, + } + if data_dict is not None: + internal_dict = Template(data_dict) + for path in paths: + if (value := internal_dict.get(path)) is not None: + if np_type := types.get(type(value).__name__): + value = np_type(value) + internal_dict[path] = {"compress": value, "strength": 3} + return internal_dict + + return None + + +def alter_dict(data_dict: Template, key: str, value: object): + """Helper function to alter a single entry in dict for parametrize.""" + if data_dict is not None: + internal_dict = Template(data_dict) + internal_dict[key] = value + return internal_dict + + return None + + def remove_from_dict(data_dict: Template, key: str, optionality: str = "optional"): """Helper function to remove a key from dict""" if data_dict is not None and key in data_dict[optionality]: @@ -1694,27 +1722,64 @@ def listify_template(data_dict: Template): id="reserved-prefix", ), pytest.param( - alter_dict( + compress_paths_in_dict( TEMPLATE, - "/ENTRY[my_entry]/NXODD_name[nxodd_name]/float_value", + [ + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/float_value" + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/number_value", + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/bool_value", + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/int_value", + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/posint_value", + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/char_value", + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/date_value", + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/type", + ], + ), + [ + "Compression for /ENTRY[my_entry]/NXODD_name[nxodd_name]/bool_value = {'compress': True, 'strength': 3} " + "should not be used for boolean values.", + "Compression for /ENTRY[my_entry]/NXODD_name[nxodd_name]/char_value = {'compress': 'just chars', 'strength': 3} " + "should not be used for string values.", + "Compression for /ENTRY[my_entry]/NXODD_name[nxodd_name]/date_value = {'compress': '2022-01-22T12:14:12.05018+00:00', 'strength': 3} " + "should not be used for string values.", + "Compression for /ENTRY[my_entry]/NXODD_name[nxodd_name]/type = {'compress': '2nd type', 'strength': 3} " + "should not be used for enumerated concepts.", + ], + id="appdef-compressed", + ), + pytest.param( + alter_dict( + alter_dict( + TEMPLATE, + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/float_value", + {"compress": np.int64(2.0), "strength": 1}, + ), + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/int_value", {"compress": np.float32(2.0), "strength": 1}, ), - [], - id="appdef-compressed-payload", + [ + "The value at /ENTRY[my_entry]/NXODD_name[nxodd_name]/int_value " + "should be one of the following Python types: " + "(, ), as defined in the " + "NXDL as NX_INT." + ], + id="appdef-compressed-wrong-type", ), pytest.param( alter_dict( TEMPLATE, - "/ENTRY[my_entry]/NXODD_name[nxodd_name]/float_value", - {"compress": np.int64(2.0), "strength": 1}, + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/int_value", + {"compress": np.float32(2.0), "strength": 11}, ), [ - "The value at /ENTRY[my_entry]/NXODD_name[nxodd_name]/float_value " + "Compression strength for /ENTRY[my_entry]/NXODD_name[nxodd_name]/int_value = " + "{'compress': 2.0, 'strength': 11} should be between 0 and 9.", + "The value at /ENTRY[my_entry]/NXODD_name[nxodd_name]/int_value " "should be one of the following Python types: " - "(, ), as defined in the " - "NXDL as NX_FLOAT." + "(, ), as defined in the " + "NXDL as NX_INT.", ], - id="appdef-compressed-payload-wrong-type", + id="appdef-compressed-invalid-strength", ), pytest.param( alter_dict( @@ -1723,7 +1788,7 @@ def listify_template(data_dict: Template): {"compress": np.int64(2), "strength": 1}, ), [], - id="baseclass-compressed-payload", + id="baseclass-compressed", ), pytest.param( alter_dict( @@ -1737,7 +1802,7 @@ def listify_template(data_dict: Template): "(, ), as defined in the " "NXDL as NX_INT." ], - id="baseclass-compressed-payload-wrong-type", + id="baseclass-compressed-wrong-type", ), ], ) From 90e0434ec49541384c746650cb7c62bbc13513ca Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Mon, 4 Aug 2025 11:01:21 +0200 Subject: [PATCH 2/8] remove copy of existing function --- tests/dataconverter/test_validation.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/tests/dataconverter/test_validation.py b/tests/dataconverter/test_validation.py index d2e3addd5..dedf4efac 100644 --- a/tests/dataconverter/test_validation.py +++ b/tests/dataconverter/test_validation.py @@ -17,7 +17,6 @@ # limitations under the License. # import logging -import random from typing import Optional import numpy as np @@ -70,16 +69,6 @@ def compress_paths_in_dict(data_dict: Template, paths=list[str]): return None -def alter_dict(data_dict: Template, key: str, value: object): - """Helper function to alter a single entry in dict for parametrize.""" - if data_dict is not None: - internal_dict = Template(data_dict) - internal_dict[key] = value - return internal_dict - - return None - - def remove_from_dict(data_dict: Template, key: str, optionality: str = "optional"): """Helper function to remove a key from dict""" if data_dict is not None and key in data_dict[optionality]: From 7c012945952857abf457d58a09ade5bce6d24e2d Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Mon, 4 Aug 2025 11:35:03 +0200 Subject: [PATCH 3/8] more informative logging --- src/pynxtools/dataconverter/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pynxtools/dataconverter/helpers.py b/src/pynxtools/dataconverter/helpers.py index 3a17f6938..6724e2746 100644 --- a/src/pynxtools/dataconverter/helpers.py +++ b/src/pynxtools/dataconverter/helpers.py @@ -223,7 +223,7 @@ def _log(self, path: str, log_type: ValidationProblem, value: Optional[Any], *ar "str": "string", "bool": "boolean", } - dtype_str = dtype_map.get(dtype, "") + dtype_str = dtype_map.get(dtype, dtype) logger.warning( f"Compression for {path} = {value} should not be used for {dtype_str} values." From ea0c65a4ab20ed86dd6c0041de2be2bfbf4096b8 Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Mon, 4 Aug 2025 11:37:43 +0200 Subject: [PATCH 4/8] mypy fixes --- .github/workflows/build_docs.yml | 1 - src/pynxtools/dataconverter/helpers.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_docs.yml b/.github/workflows/build_docs.yml index cbb484bea..215dacd40 100644 --- a/.github/workflows/build_docs.yml +++ b/.github/workflows/build_docs.yml @@ -4,7 +4,6 @@ on: push: branches: - master # Triggers deployment on push to the master branch - - restructure-docs env: python-version: 3.12 diff --git a/src/pynxtools/dataconverter/helpers.py b/src/pynxtools/dataconverter/helpers.py index 6724e2746..37e51cbc6 100644 --- a/src/pynxtools/dataconverter/helpers.py +++ b/src/pynxtools/dataconverter/helpers.py @@ -218,6 +218,7 @@ def _log(self, path: str, log_type: ValidationProblem, value: Optional[Any], *ar f"Compression for {path} = {value} should not be used for enumerated concepts." ) elif log_type == ValidationProblem.DoNotCompressStringsBoolean: + value = cast(dict, value) dtype = type(value["compress"]).__name__ dtype_map = { "str": "string", From bd31f4db0956d79c2e003e4187e977dfb3a3d5a7 Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Tue, 5 Aug 2025 11:00:34 +0200 Subject: [PATCH 5/8] add extra check for compression strength 0 --- src/pynxtools/dataconverter/helpers.py | 20 ++++++++++++++++---- tests/dataconverter/test_validation.py | 21 +++++++++++++++------ 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/src/pynxtools/dataconverter/helpers.py b/src/pynxtools/dataconverter/helpers.py index 37e51cbc6..e49e6ab04 100644 --- a/src/pynxtools/dataconverter/helpers.py +++ b/src/pynxtools/dataconverter/helpers.py @@ -83,6 +83,7 @@ class ValidationProblem(Enum): InvalidNexusTypeForNamedConcept = auto() KeysWithAndWithoutConcept = auto() InvalidCompressionStrength = auto() + CompressionStrengthZero = auto() DoNotCompressEnum = auto() DoNotCompressStringsBoolean = auto() @@ -209,7 +210,12 @@ def _log(self, path: str, log_type: ValidationProblem, value: Optional[Any], *ar logger.warning( f"The key '{path}' uses the valid concept name '{args[0]}', but there is another valid key {value} that uses the non-variadic name of the node.'" ) + elif log_type == ValidationProblem.CompressionStrengthZero: + logger.info( + f"Compression strength for {path} is 0. The value '{value['compress']}' will be written uncompressed." + ) elif log_type == ValidationProblem.InvalidCompressionStrength: + value = cast(dict, value) logger.warning( f"Compression strength for {path} = {value} should be between 0 and 9." ) @@ -251,6 +257,7 @@ def collect_and_log( if log_type not in ( ValidationProblem.UnitWithoutDocumentation, ValidationProblem.OpenEnumWithNewItem, + ValidationProblem.CompressionStrengthZero, ): self.data.add(path + str(log_type) + str(value)) @@ -812,10 +819,15 @@ def validate_data_value( if isinstance(value, dict) and set(value.keys()) == {"compress", "strength"}: compressed_value = value["compress"] - if not (0 <= value["strength"] <= value["strength"] <= 9): - collector.collect_and_log( - path, ValidationProblem.InvalidCompressionStrength, value - ) + if not (1 <= value["strength"] <= 9): + if value["strength"] == 0: + collector.collect_and_log( + path, ValidationProblem.CompressionStrengthZero, value + ) + else: + collector.collect_and_log( + path, ValidationProblem.InvalidCompressionStrength, value + ) # In this case, we remove the compression. return validate_data_value( value["compress"], nxdl_type, nxdl_enum, nxdl_enum_open, path diff --git a/tests/dataconverter/test_validation.py b/tests/dataconverter/test_validation.py index dedf4efac..7577541db 100644 --- a/tests/dataconverter/test_validation.py +++ b/tests/dataconverter/test_validation.py @@ -1758,18 +1758,26 @@ def listify_template(data_dict: Template): alter_dict( TEMPLATE, "/ENTRY[my_entry]/NXODD_name[nxodd_name]/int_value", - {"compress": np.float32(2.0), "strength": 11}, + {"compress": np.int64(2), "strength": 11}, ), [ "Compression strength for /ENTRY[my_entry]/NXODD_name[nxodd_name]/int_value = " - "{'compress': 2.0, 'strength': 11} should be between 0 and 9.", - "The value at /ENTRY[my_entry]/NXODD_name[nxodd_name]/int_value " - "should be one of the following Python types: " - "(, ), as defined in the " - "NXDL as NX_INT.", + "{'compress': 2, 'strength': 11} should be between 0 and 9.", ], id="appdef-compressed-invalid-strength", ), + pytest.param( + alter_dict( + TEMPLATE, + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/float_value", + {"compress": np.float32(2.0), "strength": 0}, + ), + [ + "Compression strength for /ENTRY[my_entry]/NXODD_name[nxodd_name]/float_value " + "is 0. The value '2.0' will be written uncompressed.", + ], + id="appdef-compressed-strength-0", + ), pytest.param( alter_dict( TEMPLATE, @@ -1814,6 +1822,7 @@ def format_error_message(msg: str) -> str: "baseclass-field-with-illegal-unit", "open-enum-with-new-item", "baseclass-open-enum-with-new-item", + "appdef-compressed-strength-0", ): with caplog.at_level(logging.INFO): assert validate_dict_against("NXtest", data_dict) From 205d8a1ddff51b88a489d6da03f6de7251d6b4d2 Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Tue, 5 Aug 2025 12:40:25 +0200 Subject: [PATCH 6/8] do not warn for compressed strings/booleans --- src/pynxtools/dataconverter/helpers.py | 43 +++++++++++--------------- tests/dataconverter/test_validation.py | 11 +------ 2 files changed, 19 insertions(+), 35 deletions(-) diff --git a/src/pynxtools/dataconverter/helpers.py b/src/pynxtools/dataconverter/helpers.py index e49e6ab04..88840be62 100644 --- a/src/pynxtools/dataconverter/helpers.py +++ b/src/pynxtools/dataconverter/helpers.py @@ -84,8 +84,7 @@ class ValidationProblem(Enum): KeysWithAndWithoutConcept = auto() InvalidCompressionStrength = auto() CompressionStrengthZero = auto() - DoNotCompressEnum = auto() - DoNotCompressStringsBoolean = auto() + # DoNotCompressStringsBoolean = auto() class Collector: @@ -219,22 +218,18 @@ def _log(self, path: str, log_type: ValidationProblem, value: Optional[Any], *ar logger.warning( f"Compression strength for {path} = {value} should be between 0 and 9." ) - elif log_type == ValidationProblem.DoNotCompressEnum: - logger.warning( - f"Compression for {path} = {value} should not be used for enumerated concepts." - ) - elif log_type == ValidationProblem.DoNotCompressStringsBoolean: - value = cast(dict, value) - dtype = type(value["compress"]).__name__ - dtype_map = { - "str": "string", - "bool": "boolean", - } - dtype_str = dtype_map.get(dtype, dtype) - - logger.warning( - f"Compression for {path} = {value} should not be used for {dtype_str} values." - ) + # elif log_type == ValidationProblem.DoNotCompressStringsBoolean: + # value = cast(dict, value) + # dtype = type(value["compress"]).__name__ + # dtype_map = { + # "str": "string", + # "bool": "boolean", + # } + # dtype_str = dtype_map.get(dtype, dtype) + + # logger.info( + # f"Compression for {path} = {value} should not be used for {dtype_str} values." + # ) def collect_and_log( self, @@ -833,13 +828,11 @@ def validate_data_value( value["compress"], nxdl_type, nxdl_enum, nxdl_enum_open, path ) - if nxdl_enum is not None: - collector.collect_and_log(path, ValidationProblem.DoNotCompressEnum, value) - - elif isinstance(compressed_value, (str, bool)): - collector.collect_and_log( - path, ValidationProblem.DoNotCompressStringsBoolean, value - ) + # TODO: Do we need to issue a warning if string/bool compression is used + # # elif isinstance(compressed_value, (str, bool)): + # collector.collect_and_log( + # path, ValidationProblem.DoNotCompressStringsBoolean, value + # ) # Apply standard validation to compressed value value["compress"] = validate_data_value( diff --git a/tests/dataconverter/test_validation.py b/tests/dataconverter/test_validation.py index 7577541db..7b28c6d4b 100644 --- a/tests/dataconverter/test_validation.py +++ b/tests/dataconverter/test_validation.py @@ -1724,16 +1724,7 @@ def listify_template(data_dict: Template): "/ENTRY[my_entry]/NXODD_name[nxodd_name]/type", ], ), - [ - "Compression for /ENTRY[my_entry]/NXODD_name[nxodd_name]/bool_value = {'compress': True, 'strength': 3} " - "should not be used for boolean values.", - "Compression for /ENTRY[my_entry]/NXODD_name[nxodd_name]/char_value = {'compress': 'just chars', 'strength': 3} " - "should not be used for string values.", - "Compression for /ENTRY[my_entry]/NXODD_name[nxodd_name]/date_value = {'compress': '2022-01-22T12:14:12.05018+00:00', 'strength': 3} " - "should not be used for string values.", - "Compression for /ENTRY[my_entry]/NXODD_name[nxodd_name]/type = {'compress': '2nd type', 'strength': 3} " - "should not be used for enumerated concepts.", - ], + [], id="appdef-compressed", ), pytest.param( From 1714a61a2a01945397c81831ed80af0b5a04f6ee Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Tue, 5 Aug 2025 12:55:49 +0200 Subject: [PATCH 7/8] mypy fix --- src/pynxtools/dataconverter/helpers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pynxtools/dataconverter/helpers.py b/src/pynxtools/dataconverter/helpers.py index 88840be62..82bf5e748 100644 --- a/src/pynxtools/dataconverter/helpers.py +++ b/src/pynxtools/dataconverter/helpers.py @@ -210,6 +210,7 @@ def _log(self, path: str, log_type: ValidationProblem, value: Optional[Any], *ar f"The key '{path}' uses the valid concept name '{args[0]}', but there is another valid key {value} that uses the non-variadic name of the node.'" ) elif log_type == ValidationProblem.CompressionStrengthZero: + value = cast(dict, value) logger.info( f"Compression strength for {path} is 0. The value '{value['compress']}' will be written uncompressed." ) From 25e18a137649cedffd7c104b83c67a7e696c9b4a Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Wed, 6 Aug 2025 10:14:45 +0200 Subject: [PATCH 8/8] add docstring --- tests/dataconverter/test_validation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/dataconverter/test_validation.py b/tests/dataconverter/test_validation.py index 7b28c6d4b..fc0fef42f 100644 --- a/tests/dataconverter/test_validation.py +++ b/tests/dataconverter/test_validation.py @@ -53,6 +53,7 @@ def set_whole_group_to_none( def compress_paths_in_dict(data_dict: Template, paths=list[str]): + """For each path, compress the value in data_dict using a strength of 3.""" types = { "int": np.int64, "float": np.float32,