Merge pull request #675 from FAIRmat-NFDI/fix_issue_670

Adding a test for issue670

Author: mkuehbach

dev-requirements.txt

@@ -1,20 +1,20 @@
 # This file was autogenerated by uv via the following command:
 #    uv pip compile --universal -p 3.12 --extra=dev --extra=docs --output-file=dev-requirements.txt pyproject.toml
-anytree==2.12.1
+anytree==2.13.0
     # via pynxtools (pyproject.toml)
-ase==3.24.0
+ase==3.25.0
     # via pynxtools (pyproject.toml)
 babel==2.17.0
     # via mkdocs-material
-backrefs==5.8
+backrefs==5.9
     # via mkdocs-material
-certifi==2025.1.31
+certifi==2025.7.14
     # via requests
 cfgv==3.4.0
     # via pre-commit
-charset-normalizer==3.4.1
+charset-normalizer==3.4.2
     # via requests
-click==8.1.8
+click==8.2.1
     # via
     #   pynxtools (pyproject.toml)
     #   click-default-group
@@ -28,13 +28,15 @@ colorama==0.4.6
     #   mkdocs
     #   mkdocs-material
     #   pytest
-contourpy==1.3.1
+contourpy==1.3.3
     # via matplotlib
-coverage==7.8.0
+coverage==7.10.1
     # via pytest-cov
 cycler==0.12.1
     # via matplotlib
-distlib==0.3.9
+debugpy==1.8.15
+    # via pynxtools (pyproject.toml)
+distlib==0.4.0
     # via virtualenv
 execnet==2.1.1
     # via pytest-xdist
@@ -44,21 +46,21 @@ flexcache==0.3
     # via pint
 flexparser==0.4
     # via pint
-fonttools==4.57.0
+fonttools==4.59.0
     # via matplotlib
 ghp-import==2.1.0
     # via mkdocs
-h5py==3.13.0
+h5py==3.14.0
     # via pynxtools (pyproject.toml)
 hjson==3.1.0
     # via
     #   mkdocs-macros-plugin
     #   super-collections
-identify==2.6.9
+identify==2.6.12
     # via pre-commit
 idna==3.10
     # via requests
-importlib-metadata==8.6.1
+importlib-metadata==8.7.0
     # via pynxtools (pyproject.toml)
 iniconfig==2.1.0
     # via pytest
@@ -69,9 +71,9 @@ jinja2==3.1.6
     #   mkdocs-material
 kiwisolver==1.4.8
     # via matplotlib
-lxml==5.3.2
+lxml==6.0.0
     # via pynxtools (pyproject.toml)
-markdown==3.7
+markdown==3.8.2
     # via
     #   markdown-include
     #   mkdocs
@@ -84,7 +86,7 @@ markupsafe==3.0.2
     # via
     #   jinja2
     #   mkdocs
-matplotlib==3.10.1
+matplotlib==3.10.3
     # via ase
 mergedeep==1.3.4
     # via
@@ -102,15 +104,15 @@ mkdocs-get-deps==0.2.0
     # via mkdocs
 mkdocs-macros-plugin==1.3.7
     # via pynxtools (pyproject.toml)
-mkdocs-material==9.6.11
+mkdocs-material==9.6.16
     # via pynxtools (pyproject.toml)
 mkdocs-material-extensions==1.3.1
     # via
     #   pynxtools (pyproject.toml)
     #   mkdocs-material
-mypy==1.15.0
+mypy==1.17.0
     # via pynxtools (pyproject.toml)
-mypy-extensions==1.0.0
+mypy-extensions==1.1.0
     # via mypy
 nodeenv==1.9.1
     # via pre-commit
@@ -124,7 +126,7 @@ numpy==1.26.4
     #   pandas
     #   scipy
     #   xarray
-packaging==24.2
+packaging==25.0
     # via
     #   matplotlib
     #   mkdocs
@@ -133,44 +135,49 @@ packaging==24.2
     #   xarray
 paginate==0.5.7
     # via mkdocs-material
-pandas==2.2.3
+pandas==2.3.1
     # via
     #   pynxtools (pyproject.toml)
     #   xarray
 pathspec==0.12.1
     # via
     #   mkdocs
     #   mkdocs-macros-plugin
-pillow==11.1.0
+    #   mypy
+pillow==11.3.0
     # via matplotlib
 pint==0.24.4
     # via pynxtools (pyproject.toml)
-platformdirs==4.3.7
+platformdirs==4.3.8
     # via
     #   mkdocs-get-deps
     #   pint
     #   virtualenv
-pluggy==1.5.0
-    # via pytest
+pluggy==1.6.0
+    # via
+    #   pytest
+    #   pytest-cov
 pre-commit==4.2.0
     # via pynxtools (pyproject.toml)
-pygments==2.19.1
-    # via mkdocs-material
-pymdown-extensions==10.14.3
+pygments==2.19.2
+    # via
+    #   mkdocs-material
+    #   pytest
+pymdown-extensions==10.16.1
     # via mkdocs-material
 pyparsing==3.2.3
     # via matplotlib
-pytest==8.3.5
+pytest==8.4.1
     # via
     #   pynxtools (pyproject.toml)
     #   pytest-cov
     #   pytest-timeout
     #   pytest-xdist
-pytest-cov==6.1.1
+pytest-cov==6.2.1
     # via pynxtools (pyproject.toml)
-pytest-timeout==2.3.1
+pytest-timeout==2.4.0
     # via pynxtools (pyproject.toml)
-pytest-xdist==3.6.1
+pytest-xdist==3.8.0
     # via pynxtools (pyproject.toml)
 python-dateutil==2.9.0.post0
     # via
@@ -189,33 +196,31 @@ pyyaml==6.0.2
     #   pre-commit
     #   pymdown-extensions
     #   pyyaml-env-tag
-pyyaml-env-tag==0.1
+pyyaml-env-tag==1.1
     # via mkdocs
-requests==2.32.3
+requests==2.32.4
     # via mkdocs-material
 ruff==0.12.5
     # via pynxtools (pyproject.toml)
-scipy==1.15.2
+scipy==1.16.1
     # via ase
 six==1.17.0
-    # via
-    #   anytree
-    #   python-dateutil
-structlog==25.2.0
+    # via python-dateutil
+structlog==25.4.0
     # via pynxtools (pyproject.toml)
 super-collections==0.5.3
     # via mkdocs-macros-plugin
-termcolor==3.0.1
+termcolor==3.1.0
     # via mkdocs-macros-plugin
 toposort==1.10
     # via pynxtools (pyproject.toml)
-types-pytz==2025.2.0.20250326
+types-pytz==2025.2.0.20250516
     # via pynxtools (pyproject.toml)
-types-pyyaml==6.0.12.20250402
+types-pyyaml==6.0.12.20250516
     # via pynxtools (pyproject.toml)
-types-requests==2.32.0.20250328
+types-requests==2.32.4.20250611
     # via pynxtools (pyproject.toml)
-typing-extensions==4.13.1
+typing-extensions==4.14.1
     # via
     #   flexcache
     #   flexparser
@@ -227,11 +232,11 @@ urllib3==2.5.0
     # via
     #   requests
     #   types-requests
-virtualenv==20.30.0
+virtualenv==20.32.0
     # via pre-commit
 watchdog==6.0.0
     # via mkdocs
-xarray==2025.3.1
+xarray==2025.7.1
     # via pynxtools (pyproject.toml)
-zipp==3.21.0
+zipp==3.23.0
     # via importlib-metadata

pyproject.toml

@@ -63,6 +63,7 @@ dev = [
     "types-pytz",
     "types-requests",
     "pre-commit",
+    "debugpy",
 ]
 convert = [
     "pynxtools[apm,ellips,em,igor,mpes,raman,spm,xps,xrd]",
@@ -136,7 +137,7 @@ select = [
     "PL", # pylint
     "UP", # pyupgrade
     # "F401", # remove unused import
-    "I001", # sort imports   
+    "I001", # sort imports
     # "NPY201", # reactivate when np>2.0 is used
 ]
 ignore = [

src/pynxtools/dataconverter/helpers.py

@@ -742,7 +742,11 @@ def is_valid_data_field(
     """
 
     accepted_types = NEXUS_TO_PYTHON_DATA_TYPES[nxdl_type]
-    # Do not count the dict as it represents a link value
+
+    if isinstance(value, dict) and set(value.keys()) == {"compress", "strength"}:
+        value = value["compress"]
+
+    # Do not count other dicts as they represent a link value
     if not isinstance(value, dict) and not is_valid_data_type(value, accepted_types):
         # try to convert string to bool
         if accepted_types[0] is bool and isinstance(value, str):

src/pynxtools/dataconverter/validation.py

@@ -16,6 +16,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+DEBUG_VALIDATION = False
 import copy
 import re
 from collections import defaultdict
@@ -24,6 +25,8 @@
 from operator import getitem
 from typing import Any, Literal, Optional, Union
 
+if DEBUG_VALIDATION:
+    import debugpy  # will connect to debugger if in debug mode
 import h5py
 import lxml.etree as ET
 import numpy as np
@@ -44,6 +47,11 @@
 from pynxtools.definitions.dev_tools.utils.nxdl_utils import get_nx_namefit
 from pynxtools.units import NXUnitSet, ureg
 
+if DEBUG_VALIDATION:
+    debugpy.debug_this_thread()
+    # set break points like this
+    # debugpy.breakpoint()
+
 
 def validate_hdf_group_against(appdef: str, data: h5py.Group):
     """
@@ -599,8 +607,10 @@ def handle_field(node: NexusNode, keys: Mapping[str, Any], prev_path: str):
         for variant in variants:
             variant_path = f"{prev_path}/{variant}"
 
-            if isinstance(keys[variant], Mapping) and not all(
-                k.startswith("@") for k in keys[variant]
+            if (
+                isinstance(keys[variant], Mapping)
+                and not all(k.startswith("@") for k in keys[variant])
+                and not list(keys[variant].keys()) == ["compress", "strength"]
             ):
                 # A field should not have a dict of keys that are _not_ all attributes,
                 # i.e. no sub-fields or sub-groups.
@@ -811,7 +821,7 @@ def is_documented(key: str, tree: NexusNode) -> bool:
             # Collection found, mark as documented
             return True
 
-        if isinstance(mapping[key], dict) and "link" in mapping[key]:
+        if isinstance(mapping[key], Mapping) and "link" in mapping[key]:
             resolved_link = _follow_link({key: mapping[key]}, "")
 
             if key not in resolved_link:

tests/dataconverter/test_validation.py

@@ -1693,6 +1693,52 @@ def listify_template(data_dict: Template):
             ],
             id="reserved-prefix",
         ),
+        pytest.param(
+            alter_dict(
+                TEMPLATE,
+                "/ENTRY[my_entry]/NXODD_name[nxodd_name]/float_value",
+                {"compress": np.float32(2.0), "strength": 1},
+            ),
+            [],
+            id="appdef-compressed-payload",
+        ),
+        pytest.param(
+            alter_dict(
+                TEMPLATE,
+                "/ENTRY[my_entry]/NXODD_name[nxodd_name]/float_value",
+                {"compress": np.int64(2.0), "strength": 1},
+            ),
+            [
+                "The value at /ENTRY[my_entry]/NXODD_name[nxodd_name]/float_value "
+                "should be one of the following Python types: "
+                "(<class 'float'>, <class 'numpy.floating'>), as defined in the "
+                "NXDL as NX_FLOAT."
+            ],
+            id="appdef-compressed-payload-wrong-type",
+        ),
+        pytest.param(
+            alter_dict(
+                TEMPLATE,
+                "/ENTRY[my_entry]/SAMPLE[sample1]]/changer_position",
+                {"compress": np.int64(2), "strength": 1},
+            ),
+            [],
+            id="baseclass-compressed-payload",
+        ),
+        pytest.param(
+            alter_dict(
+                TEMPLATE,
+                "/ENTRY[my_entry]/SAMPLE[sample1]]/changer_position",
+                {"compress": np.float32(2.0), "strength": 3},
+            ),
+            [
+                "The value at /ENTRY[my_entry]/SAMPLE[sample1]]/changer_position "
+                "should be one of the following Python types: "
+                "(<class 'int'>, <class 'numpy.integer'>), as defined in the "
+                "NXDL as NX_INT."
+            ],
+            id="baseclass-compressed-payload-wrong-type",
+        ),
     ],
 )
 def test_validate_data_dict(caplog, data_dict, error_messages, request):