Merge pull request #448 from FAIRmat-NFDI/move-nomad-examples-to-plugin

Move NOMAD examples to plugin

Author: lukaspie

.github/workflows/plugin_test.yaml

@@ -20,16 +20,21 @@ jobs:
       fail-fast: false
       matrix:
         include:
-
+          - plugin: pynxtools-apm
+            branch: main
+            tests_to_run: tests/.
           - plugin: pynxtools-ellips
             branch: main
             tests_to_run: tests/.
-          - plugin: pynxtools-raman
+          - plugin: pynxtools-em
             branch: main
             tests_to_run: tests/.
           - plugin: pynxtools-mpes
             branch: main
             tests_to_run: tests/.
+          - plugin: pynxtools-raman
+            branch: main
+            tests_to_run: tests/.
           - plugin: pynxtools-stm
             branch: main
             tests_to_run: tests/.
@@ -39,24 +44,16 @@ jobs:
           - plugin: pynxtools-xrd
             branch: main
             tests_to_run: tests/.
-          # - plugin: pynxtools-apm
-          #   branch: main
-          #   tests_to_run: tests/.
-          # - plugin: pynxtools-xrd
-          #   branch: update-tests
-          #   tests_to_run: tests/.
-          # - plugin: pynxtools-em
-          #   branch: main
-          #   tests_to_run: tests/.
+
     steps:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
           submodules: recursive
-      - name: Set up Python 3.9
+      - name: Set up Python 3.11
         uses: actions/setup-python@v5
         with:
-          python-version: 3.9
+          python-version: 3.11
       - name: Install dependencies
         run: |
           curl -LsSf https://astral.sh/uv/install.sh | sh
@@ -73,12 +70,12 @@ jobs:
           ref: ${{ matrix.branch }}
       - name: Install nomad
         run: |
-          uv pip install nomad-lab@git+https://gitlab.mpcdf.mpg.de/nomad-lab/nomad-FAIR.git
+          uv pip install nomad-lab[infrastructure]@git+https://gitlab.mpcdf.mpg.de/nomad-lab/nomad-FAIR.git
       - name: Install ${{ matrix.plugin }}
         run: |
           cd ${{ matrix.plugin }}
           uv pip install .
       - name: Run ${{ matrix.plugin }} tests
         run: |
           cd ${{ matrix.plugin }}
-          pytest ${{ matrix.tests_to_run }}
+          pytest ${{ matrix.tests_to_run }}

.github/workflows/pytest.yml

@@ -34,9 +34,9 @@ jobs:
           curl -LsSf https://astral.sh/uv/install.sh | sh
           uv pip install coverage coveralls
       - name: Install nomad
-        if: "${{ matrix.python_version != '3.8' && matrix.python_version != '3.12'}}"
+        if: "${{ matrix.python_version != '3.8'}}"
         run: |
-          uv pip install nomad-lab@git+https://gitlab.mpcdf.mpg.de/nomad-lab/nomad-FAIR.git
+          uv pip install nomad-lab[infrastructure]@git+https://gitlab.mpcdf.mpg.de/nomad-lab/nomad-FAIR.git
       - name: Install pynx
         run: |
           uv pip install ".[dev]"

.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: v0.7.0
+    rev: v0.8.0
     hooks:
       # Run the linter.
       - id: ruff

CITATION.cff

@@ -4,7 +4,7 @@ message:
   If you use this software, please cite it using the
   metadata from this file.
 type: software
-version: 0.8.2
+version: 0.9.0
 authors:
   - given-names: Sherjeel
     family-names: Shabih

MANIFEST.in

@@ -10,4 +10,5 @@ recursive-include src/pynxtools/definitions/contributed_definitions/ *.xml
 include src/pynxtools/definitions/*.xsd
 include src/pynxtools/nexus-version.txt
 include src/pynxtools/remote_definitions_url.txt
-include src/pynxtools/definitions/NXDL_VERSION
+include src/pynxtools/definitions/NXDL_VERSION
+graft src/pynxtools/nomad/examples

dev-requirements.txt

@@ -26,17 +26,17 @@ colorama==0.4.6
     #   mkdocs
     #   mkdocs-material
     #   pytest
-contourpy==1.3.0
+contourpy==1.3.1
     # via matplotlib
-coverage==7.6.4
+coverage==7.6.8
     # via pytest-cov
 cycler==0.12.1
     # via matplotlib
 distlib==0.3.9
     # via virtualenv
 filelock==3.16.1
     # via virtualenv
-fonttools==4.54.1
+fonttools==4.55.0
     # via matplotlib
 ghp-import==2.1.0
     # via mkdocs
@@ -46,7 +46,7 @@ hjson==3.1.0
     # via
     #   mkdocs-macros-plugin
     #   super-collections
-identify==2.6.1
+identify==2.6.3
     # via pre-commit
 idna==3.10
     # via requests
@@ -92,15 +92,15 @@ mkdocs-click==0.8.1
     # via pynxtools (pyproject.toml)
 mkdocs-get-deps==0.2.0
     # via mkdocs
-mkdocs-macros-plugin==1.3.6
+mkdocs-macros-plugin==1.3.7
     # via pynxtools (pyproject.toml)
-mkdocs-material==9.5.42
+mkdocs-material==9.5.46
     # via pynxtools (pyproject.toml)
 mkdocs-material-extensions==1.3.1
     # via
     #   pynxtools (pyproject.toml)
     #   mkdocs-material
-mypy==1.12.1
+mypy==1.13.0
     # via pynxtools (pyproject.toml)
 mypy-extensions==1.0.0
     # via mypy
@@ -116,7 +116,7 @@ numpy==1.26.4
     #   pandas
     #   scipy
     #   xarray
-packaging==24.1
+packaging==24.2
     # via
     #   matplotlib
     #   mkdocs
@@ -145,7 +145,7 @@ pre-commit==4.0.1
     # via pynxtools (pyproject.toml)
 pygments==2.18.0
     # via mkdocs-material
-pymdown-extensions==10.11.2
+pymdown-extensions==10.12
     # via mkdocs-material
 pyparsing==3.2.0
     # via matplotlib
@@ -154,7 +154,7 @@ pytest==8.3.3
     #   pynxtools (pyproject.toml)
     #   pytest-cov
     #   pytest-timeout
-pytest-cov==5.0.0
+pytest-cov==6.0.0
     # via pynxtools (pyproject.toml)
 pytest-timeout==2.3.1
     # via pynxtools (pyproject.toml)
@@ -177,11 +177,11 @@ pyyaml==6.0.2
     #   pyyaml-env-tag
 pyyaml-env-tag==0.1
     # via mkdocs
-regex==2024.9.11
+regex==2024.11.6
     # via mkdocs-material
 requests==2.32.3
     # via mkdocs-material
-ruff==0.7.0
+ruff==0.8.0
     # via pynxtools (pyproject.toml)
 scipy==1.14.1
     # via ase
@@ -195,7 +195,7 @@ super-collections==0.5.3
     # via mkdocs-macros-plugin
 termcolor==2.5.0
     # via mkdocs-macros-plugin
-tomli==2.0.2 ; python_full_version <= '3.11'
+tomli==2.1.0 ; python_full_version <= '3.11'
     # via coverage
 types-pytz==2024.2.0.20241003
     # via pynxtools (pyproject.toml)
@@ -211,13 +211,13 @@ urllib3==2.2.3
     # via
     #   requests
     #   types-requests
-uv==0.4.25
+uv==0.5.4
     # via pynxtools (pyproject.toml)
-virtualenv==20.27.0
+virtualenv==20.28.0
     # via pre-commit
-watchdog==5.0.3
+watchdog==6.0.0
     # via mkdocs
-xarray==2024.9.0
+xarray==2024.11.0
     # via pynxtools (pyproject.toml)
-zipp==3.20.2
+zipp==3.21.0
     # via importlib-metadata

docs/how-tos/using-pynxtools-test-framework.md

@@ -7,9 +7,7 @@ To test integration of a plugin with the `pynxtools` core system, we need to:
 ## How to write an integration test for a reader plugin with `pynxtools.testing`
 It is very simple to write a test to verify the plugin integration with `pynxtools` within the plugin's tests directory. The developer can place the test where they want, but they need to use the provided test interface from `pynxtools`. An example test for `pynxtools-FOO` (a demo plugin) plugin is given below:
 
-```python
-# test_plugin.py
-
+```python title="test_plugin.py"
 import os
 
 import pytest
@@ -51,6 +49,9 @@ def test_foo_reader(nxdl, reader_name, files_or_dir, tmp_path, caplog):
     # Use `ignore_undocumented` to skip undocumented fields
     # caplog_level can be "ERROR" or "WARNING"
     test.check_reproducibility_of_nexus()
+    # Here, you can also pass `ignore_lines` (a list) or `ignore_sections` (a dict)
+    # if you want to ignore certain lines or lines within a section in the comparison
+    # of the log files of the reference -nxs file and the one created in the test.
 ```
 
 Alongside the test data in `test/data`, it is also possible to add other types of test data inside the test directory of the plugin.
@@ -60,3 +61,82 @@ You can also pass additional parameters to `test.convert_to_nexus`:
 - `caplog_level` (str): Can be either "ERROR" (by default) or "warning". This parameter determines the level at which the caplog is set during testing. If it is "WARNING", the test will also fail if any warnings are reported by the reader.
 
 - `ignore_undocumented` (boolean): If true, the test skipts the verification of undocumented keys. Otherwise, a warning massages for undocumented keys is raised
+
+# How to write an integration test for a NOMAD example in a reader plugin
+It is also possible to ship examples for NOMAD directly with the reader plugin. As an example, `pynxtools-mpes` comes with its own NOMAD example (see [here](https://github.com/FAIRmat-NFDI/pynxtools-mpes/tree/bring-in-examples/src/pynxtools_mpes/nomad)) using the ExampleUploadEntryPoint of NOMAD (see [here](https://nomad-lab.eu/prod/v1/staging/docs/howto/plugins/example_uploads.html) for more documentation).
+
+The `testing` sub-package of `pynxtools` provides two functionalities for testing the `ExampleUploadEntryPoint` defined in a `pynxtools` plugin:
+1) Test that the ExampleUploadEntryPoint can be properly loaded
+2) Test that the schemas and files in the example folder(s) can be parsed by NOMAD
+
+We will write a test for a `pynxtools_foo_example_entrypoint` defined in the pyproject.toml file of a demo `pynxtools-FOO` (here the actual example data resides in the folder `src/pynxtools_foo/nomad/examples`):
+
+```python title="pyproject.toml"
+[project.entry-points.'nomad.plugin']
+pynxtools_foo_example = "pynxtools_foo.nomad.entrypoints:pynxtools_foo_example_entrypoint"
+```
+
+```python title="src/pynxtools_foo/nomad/nomad_example_entrypoint.py"
+from nomad.config.models.plugins import ExampleUploadEntryPoint
+
+pynxtools_foo_example_entrypoint = ExampleUploadEntryPoint(
+    title="My example upload",
+    description="""
+        This is an example upload for the pynxtools-FOO package.
+    """,
+    plugin_package="pynxtools_foo",
+    resources=["nomad/examples/*"],
+)
+```
+A test for the `pynxtools_foo_example_entrypoint` could look like this:
+```python title="test_nomad_examples.py"
+import nomad
+
+from pynxtools.testing.nomad_example import (
+    get_file_parameter,
+    parse_nomad_examples,
+    example_upload_entry_point_valid,
+)
+
+from pynxtools_foo.nomad.entrypoints import pynxtools_foo_example_entrypoint
+
+
+EXAMPLE_PATH = os.path.join(
+    os.path.dirname(__file__),
+    "..",
+    "src",
+    "pynxtools_foo",
+    "nomad",
+    "examples",
+)
+
+
+@pytest.mark.parametrize(
+    "mainfile",
+    get_file_parameter(EXAMPLE_PATH),
+)
+def test_parse_nomad_examples(mainfile):
+    """Test if NOMAD examples work."""
+    archive_dict = parse_nomad_examples(mainfile)
+    # Here, you can also implement more logic if you know the contents of the archive_dict
+
+
+@pytest.mark.parametrize(
+    ("entrypoint", "example_path"),
+    [
+        pytest.param(
+            pynxtools_foo_example_entrypoint,
+            EXAMPLE_PATH,
+            id="pynxtools_foo_example",
+        ),
+    ],
+)
+def test_example_upload_entry_point_valid(entrypoint, example_path):
+    """Test if NOMAD ExampleUploadEntryPoint works."""
+    example_upload_entry_point_valid(
+        entrypoint=entrypoint,
+        example_path=example_path,
+    )
+
+```
+

pyproject.toml

@@ -96,6 +96,7 @@ xrd = [
 nexus_parser = "pynxtools.nomad.entrypoints:nexus_parser"
 nexus_schema = "pynxtools.nomad.entrypoints:nexus_schema"
 nexus_data_converter = "pynxtools.nomad.entrypoints:nexus_data_converter"
+iv_temp_example = "pynxtools.nomad.entrypoints:iv_temp_example"
 
 [project.scripts]
 read_nexus = "pynxtools.nexus.nexus:main"
@@ -128,7 +129,7 @@ select = [
     "E", # pycodestyle
     "W", # pycodestyle
     "PL", # pylint
-    "NPY201",
+    # "NPY201", # reactivate when np>2.0 is used
 ]
 ignore = [
     "E501", # Line too long ({width} > {limit} characters)

src/pynxtools/nomad/entrypoints.py

@@ -1,5 +1,26 @@
+#
+# Copyright The NOMAD Authors.
+#
+# This file is part of NOMAD. See https://nomad-lab.eu for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
 try:
-    from nomad.config.models.plugins import ParserEntryPoint, SchemaPackageEntryPoint
+    from nomad.config.models.plugins import (
+        ParserEntryPoint,
+        SchemaPackageEntryPoint,
+        ExampleUploadEntryPoint,
+    )
 except ImportError as exc:
     raise ImportError(
         "Could not import nomad package. Please install the package 'nomad-lab'."
@@ -43,3 +64,14 @@ def load(self):
     mainfile_name_re=r".*\.nxs",
     mainfile_mime_re="application/x-hdf5",
 )
+
+iv_temp_example = ExampleUploadEntryPoint(
+    title="Sensor Scan - IV Temperature Curve",
+    category="FAIRmat examples",
+    description="""
+        This example shows users how to take data from a Python framework and map it out to a Nexus application definition for IV Temperature measurements, [`NXiv_temp`](https://fairmat-nfdi.github.io/nexus_definitions/classes/contributed_definitions/NXiv_temp.html).
+        We use the Nexus ELN features of NOMAD to generate a Nexus file.
+    """,
+    plugin_package="pynxtools",
+    resources=["nomad/examples/iv_temp/*"],
+)