align pynxtools.nomad module with recommended nomad plugin structure

.github/workflows/publish.yml

@@ -82,7 +82,7 @@ jobs:
       - name: Build package
         run: |
           git reset --hard HEAD
-          rm -f src/pynxtools/nomad/nxs_metainfo_package_*.json
+          rm -f src/pynxtools/nomad/schema_packages/nxs_metainfo_package_*.json
           export PYNXTOOLS_BUILD_PACKAGE=1
           uv run --extra nomad scripts/generate_package.py
           uv build
@@ -117,7 +117,7 @@ jobs:
           EXPECTED=$(python -c "from pynxtools import get_nexus_version; print(f'nxs_metainfo_package_{get_nexus_version()}.json')")
 
           # find the JSON file
-          FOUND_FILE=$(find "$SITE_PACKAGES/pynxtools/nomad" -name 'nxs_metainfo_package_*.json' -print -quit)
+          FOUND_FILE=$(find "$SITE_PACKAGES/pynxtools/nomad/schema_packages" -name 'nxs_metainfo_package_*.json' -print -quit)
 
           if [ -z "$FOUND_FILE" ]; then
               echo "ERROR: nxs_metainfo_package_*.json not found in installed package."

.gitignore

@@ -206,6 +206,6 @@ cython_debug/
 !src/pynxtools/units/constants_en.txt
 !src/pynxtools/units/default_en.txt
 build/
-src/pynxtools/nomad/nxs_metainfo_*.json
+src/pynxtools/nomad/schema_packages/nxs_metainfo_*.json
 nexusparser.egg-info/PKG-INFO
 .python-version

MANIFEST.in

@@ -12,5 +12,5 @@ include src/pynxtools/nexus-version.txt
 include src/pynxtools/remote_definitions_url.txt
 include src/pynxtools/definitions/NXDL_VERSION
 include src/pynxtools/units/*.txt
-graft src/pynxtools/nomad/examples
-include src/pynxtools/nomad/nxs_metainfo_package_*.json
+graft src/pynxtools/nomad/example_uploads/iv_temp_example
+include src/pynxtools/nomad/schema_packages/nxs_metainfo_package_*.json

pyproject.toml

@@ -102,11 +102,11 @@ xrd = [
 ]
 
 [project.entry-points.'nomad.plugin']
-nexus_parser = "pynxtools.nomad.entrypoints:nexus_parser"
-nexus_schema = "pynxtools.nomad.entrypoints:nexus_schema"
-nexus_data_converter = "pynxtools.nomad.entrypoints:nexus_data_converter"
-nexus_app = "pynxtools.nomad.entrypoints:nexus_app"
-simple_nexus_example = "pynxtools.nomad.entrypoints:simple_nexus_example"
+nexus_parser = "pynxtools.nomad.parsers:nexus_parser"
+nexus_schema = "pynxtools.nomad.schema_packages:nexus_schema"
+nexus_data_converter = "pynxtools.nomad.schema_packages:nexus_data_converter"
+nexus_app = "pynxtools.nomad.apps:nexus_app"
+simple_nexus_example = "pynxtools.nomad.example_uploads:simple_nexus_example"
 
 [project.scripts]
 read_nexus = "pynxtools.nexus.nexus:main"

scripts/clean_old_packages.sh

@@ -5,10 +5,10 @@ project_dir=$(dirname "$(dirname "$(realpath "$0")")")
 cd "$project_dir" || exit 1
 
 # Get the full path of the file to keep
-nxs_file=$(python3 -c "from pynxtools.nomad.utils import get_package_filepath; print(get_package_filepath())")
+nxs_file=$(python3 -c "from pynxtools.nomad import get_package_filepath; print(get_package_filepath())")
 
 # Define the target directory
-target_dir="src/pynxtools/nomad"
+target_dir="src/pynxtools/nomad/schema_packages"
 
 # Delete all matching files except nxs_file
 for file in "$target_dir"/nxs_metainfo_package_*.json; do

scripts/generate_package.py

@@ -2,7 +2,7 @@
 import os
 
 try:
-    from pynxtools.nomad.utils import get_package_filepath
+    from pynxtools.nomad import get_package_filepath
 except ImportError as e:
     raise ImportError(
         "The 'pynxtools' package is required but not installed. "
@@ -14,6 +14,6 @@
 
 if not os.path.exists(nxs_filepath):
     logger.info(f"Generating NeXus package at {nxs_filepath}.")
-    import pynxtools.nomad.schema  # noqa: F401
+    import pynxtools.nomad.schema_packages.schema  # noqa: F401
 else:
     logger.info(f"NeXus package already existed at {nxs_filepath}.")

src/pynxtools/nomad/__init__.py

@@ -0,0 +1,169 @@
+#
+# Copyright The NOMAD Authors.
+#
+# This file is part of NOMAD. See https://nomad-lab.eu for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import os
+from pathlib import Path
+from typing import Optional
+
+import numpy as np
+
+from pynxtools import get_nexus_version
+
+try:
+    from nomad import config
+    from nomad.metainfo.data_type import (
+        Bytes,
+        Datetime,
+        m_bool,
+        m_complex128,
+        m_float64,
+        m_int,
+        m_int64,
+        m_str,
+    )
+except ImportError as exc:
+    raise ImportError(
+        "Could not import nomad package. Please install the package 'nomad-lab'."
+    ) from exc
+
+REPLACEMENT_FOR_NX = ""
+
+# This is a list of NeXus group names that are not allowed because they are defined as quantities in the BaseSection class.
+UNALLOWED_GROUP_NAMES = {"name", "datetime", "lab_id", "description"}
+
+NX_TYPES = {  # Primitive Types,  'ISO8601' is the only type not defined here
+    "NX_COMPLEX": m_complex128,
+    "NX_FLOAT": m_float64,
+    "NX_CHAR": m_str,
+    "NX_BOOLEAN": m_bool,
+    "NX_INT": m_int64,
+    "NX_UINT": m_int64,
+    "NX_NUMBER": m_float64,
+    "NX_POSINT": m_int64,
+    "NX_BINARY": Bytes,
+    "NX_DATE_TIME": Datetime,
+    "NX_CHAR_OR_NUMBER": m_float64,  # TODO: fix this mapping
+}
+
+FIELD_STATISTICS: dict[str, list] = {
+    "suffix": ["__mean", "__std", "__min", "__max", "__size", "__ndim"],
+    "function": [np.mean, np.std, np.min, np.max, np.size, np.ndim],
+    "type": [np.float64, np.float64, None, None, np.int32, np.int32],
+    "mask": [True, True, True, True, False, False],
+}
+
+
+def _rename_classes_in_nomad(nx_name: str) -> str:
+    """
+    Modify group names that conflict with NOMAD due to being defined as quantities
+    in the BaseSection class by appending '__group' to those names.
+
+    Some quantities names names are reserved in the BaseSection class (or even higher up in metainfo),
+    and thus require renaming to avoid collisions.
+
+    Args:
+        nx_name (str): The original group name.
+
+    Returns:
+        Optional[str]: The modified group name with '__group' appended if it's in
+        UNALLOWED_GROUP_NAMES, or the original name if no change is needed.
+    """
+    return nx_name + "__group" if nx_name in UNALLOWED_GROUP_NAMES else nx_name
+
+
+def _rename_nx_for_nomad(
+    name: str,
+    is_group: bool = False,
+    is_field: bool = False,
+    is_attribute: bool = False,
+) -> str | None:
+    """
+    Rename NXDL names for compatibility with NOMAD, applying specific rules
+    based on the type of the NeXus concept. (group, field, or attribute).
+
+    - NXobject is unchanged.
+    - NX-prefixed names (e.g., NXdata) are renamed by replacing 'NX' with a custom string.
+    - Group names are passed to _rename_classes_in_nomad(), and the result is capitalized.
+    - Fields and attributes have '__field' or '__attribute' appended, respectively.
+
+    Args:
+        name (str): The NXDL name.
+        is_group (bool): Whether the name represents a group.
+        is_field (bool): Whether the name represents a field.
+        is_attribute (bool): Whether the name represents an attribute.
+
+    Returns:
+        Optional[str]: The renamed NXDL name, with group names capitalized,
+        or None if input is invalid.
+    """
+    if name and name.startswith("NX"):
+        name = REPLACEMENT_FOR_NX + name[2:]
+        name = name[0].upper() + name[1:]
+
+    if name[0] in "0123456789":
+        name = f"_{name}"
+
+    if is_group:
+        name = _rename_classes_in_nomad(name)
+    elif is_field:
+        name += "__field"
+    elif is_attribute:
+        pass
+    return name
+
+
+def get_quantity_base_name(quantity_name):
+    return (
+        quantity_name[:-7]
+        if quantity_name.endswith("__field") and quantity_name[-8] != "_"
+        else quantity_name
+    )
+
+
+PACKAGE_DIR = Path(__file__).resolve().parent / "schema_packages"
+CACHE_DIR = Path(config.fs.tmp) / "pynxtools"
+
+
+def get_package_filepath() -> Path:
+    """Return the path to the NeXus metainfo package JSON file.
+
+    Resolution order:
+
+    1. If ``PYNXTOOLS_BUILD_PACKAGE`` is ``"1"``, always return the path inside
+       ``PACKAGE_DIR`` (ensures inclusion in built distributions).
+    2. If the file already exists in ``PACKAGE_DIR``, return that path.
+    3. Otherwise, return the corresponding path inside ``CACHE_DIR`` for
+       development or first-time generation.
+
+    Returns:
+        Path: Resolved location for reading or writing the JSON file.
+    """
+    filename = f"nxs_metainfo_package_{get_nexus_version()}.json"
+
+    # 1. Build-mode override (forces packaging the file)
+    if os.environ.get("PYNXTOOLS_BUILD_PACKAGE") == "1":
+        return PACKAGE_DIR / filename
+
+    # 2. Use packaged file if it exists
+    packaged = PACKAGE_DIR / filename
+    if packaged.exists():
+        return packaged
+
+    # 3. Otherwise store in cache dir
+    # create parent directory only if we need to write
+    CACHE_DIR.mkdir(parents=True, exist_ok=True)
+    return CACHE_DIR / filename

src/pynxtools/nomad/entrypoints.py → src/pynxtools/nomad/apps/__init__.py

@@ -16,68 +16,23 @@
 # limitations under the License.
 #
 try:
-    from nomad.config.models.plugins import (
-        AppEntryPoint,
-        ExampleUploadEntryPoint,
-        ParserEntryPoint,
-        SchemaPackageEntryPoint,
+    from nomad.config.models.plugins import AppEntryPoint
+    from nomad.config.models.ui import (
+        App,
+        Column,
+        Menu,
+        MenuItemHistogram,
+        MenuItemPeriodicTable,
+        MenuItemTerms,
+        MenuSizeEnum,
+        SearchQuantities,
     )
 except ImportError as exc:
     raise ImportError(
         "Could not import nomad package. Please install the package 'nomad-lab'."
     ) from exc
 
-
-class NexusParserEntryPoint(ParserEntryPoint):
-    def load(self):
-        from pynxtools.nomad.parser import NexusParser
-
-        return NexusParser(**self.dict())
-
-
-class NexusSchemaEntryPoint(SchemaPackageEntryPoint):
-    def load(self):
-        from pynxtools.nomad.schema import nexus_metainfo_package
-
-        return nexus_metainfo_package
-
-
-class NexusDataConverterEntryPoint(SchemaPackageEntryPoint):
-    def load(self):
-        from pynxtools.nomad.dataconverter import m_package
-
-        return m_package
-
-
-nexus_schema = NexusSchemaEntryPoint(
-    name="NeXus",
-    description="The NeXus metainfo package.",
-)
-
-nexus_data_converter = NexusDataConverterEntryPoint(
-    name="NeXus Dataconverter",
-    description="The NeXus dataconverter to convert data into the NeXus format.",
-)
-
-nexus_parser = NexusParserEntryPoint(
-    name="pynxtools parser",
-    description="A parser for nexus files.",
-    mainfile_name_re=r".*\.nxs",
-    mainfile_mime_re="application/x-hdf*",
-)
-
-from nomad.config.models.ui import (
-    App,
-    Column,
-    Menu,
-    MenuItemHistogram,
-    MenuItemPeriodicTable,
-    MenuItemTerms,
-    MenuSizeEnum,
-    SearchQuantities,
-)
-
-schema = "pynxtools.nomad.schema.Root"
+schema = "pynxtools.nomad.schema_packages.schema.Root"
 
 
 nexus_app = AppEntryPoint(
@@ -356,23 +311,3 @@ def load(self):
         },
     ),
 )
-
-simple_nexus_example = ExampleUploadEntryPoint(
-    title="Simple NeXus Example",
-    category="NeXus Experiment Examples",
-    description="""
-        Sensor Scan - IV Temperature Curve
-        This example shows how experimental data can be mapped to a Nexus application definition.
-        Here, data from an IV Temperature measurements as taken by a Python framework is
-        converted to [`NXiv_temp`](https://fairmat-nfdi.github.io/nexus_definitions/classes/contributed_definitions/NXiv_temp.html).
-        We also demonstrate the use of Nexus ELN features of NOMAD to add further details
-        which were not provided by the data acquisition software.
-        This example demonstrates how
-        - a NOMAD ELN can be built and its content can be written to an RDM platform agnostic yaml format
-        - NOMAD ELN can be used to combine ELN data with experiment data and export them to NeXus
-        - NeXus data is represented as an Entry with searchable quantities in NOMAD
-        - NORTH tools can be used to work with data in NOMAD uploads
-    """,
-    plugin_package="pynxtools",
-    resources=["nomad/examples/*"],
-)

src/pynxtools/nomad/example_uploads/__init__.py

@@ -0,0 +1,44 @@
+#
+# Copyright The NOMAD Authors.
+#
+# This file is part of NOMAD. See https://nomad-lab.eu for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+try:
+    from nomad.config.models.plugins import ExampleUploadEntryPoint
+except ImportError as exc:
+    raise ImportError(
+        "Could not import nomad package. Please install the package 'nomad-lab'."
+    ) from exc
+
+simple_nexus_example = ExampleUploadEntryPoint(
+    title="Simple NeXus Example",
+    category="NeXus Experiment Examples",
+    description="""
+        Sensor Scan - IV Temperature Curve
+        This example shows how experimental data can be mapped to a Nexus application definition.
+        Here, data from an IV Temperature measurements as taken by a Python framework is
+        converted to [`NXiv_temp`](https://fairmat-nfdi.github.io/nexus_definitions/classes/contributed_definitions/NXiv_temp.html).
+        We also demonstrate the use of Nexus ELN features of NOMAD to add further details
+        which were not provided by the data acquisition software.
+        This example demonstrates how
+        - a NOMAD ELN can be built and its content can be written to an RDM platform agnostic yaml format
+        - NOMAD ELN can be used to combine ELN data with experiment data and export them to NeXus
+        - NeXus data is represented as an Entry with searchable quantities in NOMAD
+        - NORTH tools can be used to work with data in NOMAD uploads
+    """,
+    plugin_package="pynxtools",
+    resources=["nomad/example_uploads/iv_temp_example/*"],
+)