Ontology Service Integration

.cspell/custom-dictionary.txt

@@ -13,6 +13,8 @@ Downsample
 Downsampled
 Draxl
 ELECTRONANALYZER
+ESRFET
+ESRF
 Emminger
 Florian
 Forschungsgemeinschaft
@@ -43,6 +45,7 @@ Namefits
 Namefitting
 OPCPA
 ORCID
+OWLREADY
 PATHCONV
 PDBX
 PYNXTOOLS
@@ -59,6 +62,7 @@ Scheidgen
 Shabih
 Sherjeel
 Tommaso
+testdata
 UNALLOWED
 Uniquified
 Uniquifies
@@ -95,8 +99,8 @@ edgeitems
 ekey
 electronanalyzer
 elems
-ellips
 elist
+ellips
 ellipsometry
 emittance
 execinfo
@@ -158,6 +162,7 @@ nexusformat
 nodemixin
 nonvariadic
 nsmap
+nxclass
 nxcollection
 nxdata
 nxdl
@@ -166,14 +171,18 @@ nxentry
 optionalities
 orcid
 otherfile
+owlready
 oxidising
+panet
 pathtovalidate
 plottable
 posint
 printoptions
 punx
 pynxtools
 raman
+rdfs
+rdfxml
 redef
 reqs
 requiredness

.gitignore

@@ -193,7 +193,7 @@ cython_debug/
 
 
 # Custom settings
-
+.volumes/
 .mypy_cache/
 .pyenv
 *.pyc
@@ -205,7 +205,10 @@ cython_debug/
 !src/pynxtools/remote_definitions_url.txt
 !src/pynxtools/units/constants_en.txt
 !src/pynxtools/units/default_en.txt
+src/pynxtools/NeXusOntology/ontology/NeXusOntology_c*.owl
+src/pynxtools/NeXusOntology/ontology/NeXusOntology_full_c*.owl
+src/pynxtools/NeXusOntology/ontology/NeXusOntology_full_*_inferred.owl
 build/
 src/pynxtools/nomad/nxs_metainfo_*.json
 nexusparser.egg-info/PKG-INFO
-.python-version
+.python-version

.gitmodules

@@ -1,3 +1,9 @@
 [submodule "src/pynxtools/definitions"]
 	path = src/pynxtools/definitions
 	url = https://github.com/FAIRmat-NFDI/nexus_definitions.git
+
+[submodule "src/pynxtools/NeXusOntology"]
+	path = src/pynxtools/NeXusOntology
+	url = https://github.com/FAIRmat-NFDI/NeXusOntology.git
+	branch = oscars-project
+	fetchRecurseSubmodules = false

MANIFEST.in

@@ -14,3 +14,6 @@ include src/pynxtools/definitions/NXDL_VERSION
 include src/pynxtools/units/*.txt
 graft src/pynxtools/nomad/examples
 include src/pynxtools/nomad/nxs_metainfo_package_*.json
+include src/pynxtools/NeXusOntology/script/*.py
+include src/pynxtools/NeXusOntology/script/*.py
+src/pynxtools/NeXusOntology/ontology/NeXusOntology_full_*_inferred.owl

cspell.json

@@ -10,7 +10,8 @@
         "__pycache__",
         "dist",
         "build",
-        "src/pynxtools/definitions"
+        "src/pynxtools/definitions",
+        "src/pynxtools/NeXusOntology"
     ],
   "dictionaryDefinitions": [
     {

docs/how-tos/pynxtools/ontology-service.md

@@ -0,0 +1,83 @@
+# Using the ontology service in pynxtools
+
+!!! info "This is a how-to guide for using the ontology service. If you want to learn more about how `ontology service` works in `pynxtools`, please visit the [explanation](../../learn/pynxtools/ontology-service.md) page."
+
+## Prerequisites
+
+- If you plan to work on `pynxtools` and the ontology service locally, follow the [development guide](../../tutorial/contributing.md). Ensure the NeXusOntology and definitions submodules [are initialized](../../tutorial/contributing.md#development-installation).
+- Python 3.10+
+- You will first need to install pynxtools and its dependencies (see [pyproject.toml](https://github.com/FAIRmat-NFDI/pynxtools/blob/master/pyproject.toml)), including owlready2, pygit2, fastapi. For more information about the installation, see [installation guide](../../tutorial/installation.md)
+
+!!! note "NeXusOntology and definitions submodules"
+    === "Using pynxtools (standard installation)"
+        If you are simply using `pynxtools` as a package, the NeXusOntology and definitions submodules are included automatically during installation. Follow the [installation guide](../../tutorial/installation.md) to get started.
+
+    === "Developing the ontology service"
+        If you are contributing to or developing the ontology service, you need to manually initialize the submodules. See the [development guide](../../tutorial/contributing.md#development-installation) for instructions on setting up the development environment.
+
+## Getting started
+
+### Importing and using the service
+
+The main entry point is the FastAPI app defined in [`pynxtools.nomad.apis.ontology_service`](https://github.com/FAIRmat-NFDI/pynxtools/blob/ontology-service/src/pynxtools/nomad/apis/ontology_service.py). You can import and use the service as follows:
+
+```python
+from pynxtools.nomad.apis.ontology_service import app, load_ontology, fetch_superclasses
+```
+
+## Configuring ontology imports
+
+The ontologies used by the ontology service in `pynxtools` are configurable via the [`nomad.yaml`](https://nomad-lab.eu/prod/v1/docs/howto/develop/setup.html#nomadyaml) configuration file. This allows you to control exactly which ontologies are loaded and reasoned over.
+
+To specify which ontologies to import, add their URLs under the `plugins.options.pynxtools.nomad.apis:ontology_service.imports` section in your `nomad.yaml`:
+
+```yaml
+plugins:
+  options:
+    pynxtools.nomad.apis:ontology_service:
+      imports:
+        - "https://raw.githubusercontent.com/pan-ontologies/esrf-ontologies/refs/heads/oscars-deliverable-2/ontologies/esrfet/ESRFET.owl"
+        - "http://purl.org/pan-science/PaNET/PaNET.owl"
+```
+
+By default, the [ESRFET](https://github.com/pan-ontologies/esrf-ontologies) and [PaNET](https://bioportal.bioontology.org/ontologies/PANET) ontology are imported, like in the example.
+
+**Important:**  
+Even if an ontology (e.g., [ESRFET](https://github.com/pan-ontologies/esrf-ontologies)) references another ontology (e.g., [PaNET](https://bioportal.bioontology.org/ontologies/PANET)) via `owl:imports`, you must still list both URLs explicitly in the imports. The ontology service only loads the ontologies specified in the configuration and does not automatically follow `owl:imports` statements.
+
+For details on using `pynxtools` as a NOMAD plugin, refer to the [development guide](../pynxtools/../../tutorial/contributing.md#developing-pynxtools-as-a-nomad-plugin).
+
+### Minimal working example
+
+You can extract superclasses for a NeXus class using the ontology service's HTTP API endpoint. Here's an example using Python's `requests` library:
+
+```python
+import requests
+
+# Replace with the actual running service URL
+base_url = "http://localhost:8000/nomad-oasis/"
+class_name = "NXmpes_arpes"
+response = requests.get(f"{base_url}/superclasses/{class_name}")
+if response.status_code == 200:
+  superclasses = response.json().get("superclasses", [])
+  print(superclasses)
+else:
+  print(f"Error: {response.status_code} - {response.text}")
+```
+
+This endpoint returns a JSON object with the list of superclasses for the given NeXus class name, as used internally in [pynxtools](https://github.com/FAIRmat-NFDI/pynxtools/blob/ontology-service/src/pynxtools/nomad/schema.py).
+
+## How it works in NOMAD
+
+When you upload a NeXus file to NOMAD, the ontology service is automatically triggered during data processing. Here's what happens:
+
+1. **Triggering**: During normalization, NOMAD reads the `definition__field` from NeXus entry (e.g., `NXmpes_arpes`).
+
+    ![Processing NeXus files in NOMAD](../../assets/ontology-service-trigger.png){ width="800" }
+    ![Reading definition field](../../assets/entry_definition__field.png){ width="800" }
+
+2. **Querying**: The service loads the ontology (or generates it if not already present) and retrieves all superclasses for that application definition.
+
+3. **Storing results**: The retrieved superclasses are stored in `results.eln.methods` in the NOMAD archive.
+
+    ![Ontology results in NOMAD](../../assets/superclasses_results.png){ width="800" }

docs/index.md

@@ -50,6 +50,7 @@ We are offering a small guide to getting started with NeXus, `pynxtools`, and NO
 - [Running `pynxtools` tests in parallel](how-tos/pynxtools/run-tests-in-parallel.md)
 - [Using Python to create NeXus files](how-tos/pynxtools/create-nexus-files-by-python.md)
 - [Validation of NeXus files](how-tos/pynxtools/validate-nexus-files.md)
+- [Use the ontology service](how-tos/pynxtools/ontology-service.md)
 
 </div>
 
@@ -68,6 +69,7 @@ We are offering a small guide to getting started with NeXus, `pynxtools`, and NO
 - [Data conversion in `pynxtools`](learn/pynxtools/dataconverter-and-readers.md)
 - [Validation of NeXus files](learn/pynxtools/nexus-validation.md)
 - [The `MultiFormatReader` as a reader superclass](learn/pynxtools/multi-format-reader.md)
+- [Introduction to the ontology service](learn/pynxtools/ontology-service.md)
 
 </div>
 <div markdown="block">

docs/learn/pynxtools/ontology-service.md

@@ -0,0 +1,72 @@
+# Understanding the ontology service in pynxtools
+
+!!! info "This is a learn guide for using the ontology service. If you want to learn more about how to use `ontology service` in `pynxtools`, please visit the [how-to guide](../../how-tos/pynxtools/ontology-service.md)."
+
+## Introduction
+
+The ontology service in `pynxtools` provides a [FastAPI](https://fastapi.tiangolo.com/)-based app for querying the [NeXusOntology](https://github.com/nexusformat/NeXusOntology). This service enables users to retrieve semantic contextualization—such as superclasses for NeXus application definitions—and makes datasets of different experimental techniques findable in NOMAD.
+
+The NeXus ontology integrates two key experimental technique ontologies:
+
+- **[PaN Experimental Technique Ontology (PaNET)](https://bioportal.bioontology.org/ontologies/PANET)**: Photon and Neutron Experimental Techniques ontology (PaNET) provides a standardized vocabulary for describing experimental techniques used at photon and neutron research facilities, enabling consistent categorization and discovery of scientific data across different institutions.
+
+- **[ESRF Experimental Technique Ontology (ESRFET)](https://github.com/pan-ontologies/esrf-ontologies)**: An ontology developed by the European Synchrotron Radiation Facility (ESRF) to classify and describe experimental techniques specific to synchrotron radiation science. It complements PaNET by providing more granular terms for synchrotron-based methods.
+
+By using these ontologies, the service maps terms that are defined in NeXus application definitions to standardized terms for experimental techniques, improving data interoperability and enabling semantic search capabilities within NOMAD.
+
+## How it fits in pynxtools
+
+The `ontology service` only operates when `pynxtools` is run within `NOMAD`. It is registered as an [`APIEntryPoint`](https://nomad-lab.eu/prod/v1/docs/reference/config.html#apientrypoint), which allows NOMAD to automatically discover and mount the FastAPI routes at startup. The service exposes the [NeXusOntology](https://github.com/nexusformat/NeXusOntology) to users and connected systems.
+
+## Core concepts
+
+- **Ontology**: A formal representation of concepts (classes), relationships, and properties, typically serialized as [OWL](https://www.w3.org/OWL/) files.
+- **Name of an application definition**: A symbol describing an experimental technique in NeXus (e.g., `NXmpes_arpes`).
+- **Relation**: A connection between entities, such as a superclass/subclass relationship. 
+- **Inferred ontology**: An ontology file that has been processed by a reasoner to infer additional relationships and properties between classes, beyond those explicitly defined in the original ontology.
+
+Ontologies are represented as OWL files (e.g., `NeXusOntology_full_<commit>_inferred.owl`) and loaded using `owlready2`.
+
+## Architecture & design
+
+- **Main modules**:
+    - `pynxtools.nomad.apis.ontology_service`: FastAPI app, core logic for loading and querying ontologies.
+    - `pynxtools.NeXusOntology.script.generate_ontology`: Generates ontology files from NeXus definitions.
+    - `pynxtools.nomad.schema`: Initializes [NOMAD Metainfo](https://nomad-lab.eu/prod/v1/docs/explanation/data.html#schema) and schema integration..
+- **Key classes/functions**:
+    - `load_ontology()`: Loads the inferred ontology file.
+    - `fetch_superclasses(ontology, class_name)`: Retrieves superclasses for a given NeXus class.
+    - `ensure_ontology_file()`: Ensures an ontology file is present and up-to-date.
+- **Data flow**:
+    - On startup, the service verifies whether the inferred ontology file exists; if absent, it runs the reasoner and generates the inferred ontology file.
+    - Inferred ontology is loaded via `owlready2`.
+    - API endpoints query this inferred ontology for relationships and metadata.
+
+## Extensibility points
+
+- Extend FastAPI routes in [`ontology_service.py`](https://github.com/FAIRmat-NFDI/pynxtools/blob/ontology-service/src/pynxtools/nomad/apis/ontology_service.py) for new queries.
+
+## Examples
+
+```python
+import requests
+
+# Replace with the actual running service URL
+base_url = "http://localhost:8000/nomad-oasis/"
+class_name = "NXmpes_arpes"
+response = requests.get(f"{base_url}/superclasses/{class_name}")
+if response.status_code == 200:
+  superclasses = response.json().get("superclasses", [])
+  print(superclasses)
+else:
+  print(f"Error: {response.status_code} - {response.text}")
+```
+
+## Glossary
+
+- **NeXus**: A common data format for neutron, X-ray, and muon science.
+- **OWL**: Web Ontology Language, used for representing ontologies.
+- **Ontology**: Structured representation of concepts and relationships.
+- **Superclass**: A parent class in the ontology hierarchy.
+- **Reasoner**: Tool for inferring new relationships in an ontology.
+- **NOMAD**: The FAIRmat NOMAD project for materials data.

docs/tutorial/contributing.md

@@ -60,6 +60,13 @@ git submodule update --init --recursive --jobs=4
 
 Note that we are using the NeXus definitions as a [Git submodule](https://git-scm.com/book/en/v2/Git-Tools-Submodules). The last two lines initiate the submodule and upgrade it to match the first used in pynxtools.
 
+For the [ontology service](../learn/pynxtools/ontology-service.md), adding the [NeXusOntology](https://github.com/FAIRmat-NFDI/NeXusOntology/tree/oscars-project) as a git submodule is required. Here, it is recommended to use the sparse checkout:
+
+```bash
+git sparse-checkout init --no-cone
+git sparse-checkout set "/*" '!ontology/NeXusOntology.owl' '!ontology/NeXusOntology_full.owl' '!ontology/NeXusOntology_full_testdata.owl'
+```
+
 Next, we install the package in editable mode (together with its dependencies):
 
 === "uv"
@@ -79,7 +86,7 @@ Next, we install the package in editable mode (together with its dependencies):
 
 ### Linting and formatting
 
-We are using ruff and mypy for linting, formatting, and type checking. It is recommended to use the [pre-commit hook](https://pre-commit.com/#intro) available for ruff which formats the code and checks the linting before actually making an actual Git commit.
+We are using ruff and mypy for linting, formatting, and type checking. It is recommended to use the [pre-commit hook](https://pre-commit.com/#intro) available for ruff which formats the code and checks the linting before making an actual Git commit.
 
 Install the precommit by running
 
@@ -99,7 +106,7 @@ pytest -sv tests
 
 ### Editing the documentation
 
-We are using [`mkdocs](https://www.mkdocs.org/) for the documentation. If you edit the documentation, you can build it locally. For this, you need to install an additional set of dependencies:
+We are using [mkdocs](https://www.mkdocs.org/) for the documentation. If you edit the documentation, you can build it locally. For this, you need to install an additional set of dependencies:
 
 === "uv"
 
@@ -140,7 +147,7 @@ A number of examples exist which document how the tools can be used. For a stand
 
 ### Contributing to the package on Github
 
-Once you are happy with the changes, please commit them on a separate branch and create a pull request on GitHub. We run a number of GitHub actions that check the correct linting, run the tests in an isolated environment, and build the documentation. Once these pass and a peer review of the code has occurred, your code will be accepted.
+Once you are happy with the changes, please commit them on a separate branch and create a [pull request on GitHub](https://github.com/FAIRmat-NFDI/pynxtools/pulls). We run a number of GitHub actions that check the correct linting, run the tests in an isolated environment, and build the documentation. Once these pass and a peer review of the code has occurred, your code will be accepted.
 
 ## Developing pynxtools as a NOMAD plugin
 

mkdocs.yaml

@@ -25,6 +25,7 @@ nav:
       - how-tos/pynxtools/run-tests-in-parallel.md
       - how-tos/pynxtools/create-nexus-files-by-python.md
       - how-tos/pynxtools/validate-nexus-files.md
+      - how-tos/pynxtools/ontology-service.md
   - Learn:
     - NeXus:
       - learn/nexus/nexus-primer.md
@@ -34,6 +35,7 @@ nav:
       - learn/pynxtools/dataconverter-and-readers.md
       - learn/pynxtools/nexus-validation.md
       - learn/pynxtools/multi-format-reader.md
+      - learn/pynxtools/ontology-service.md
   - Reference:
     - reference/definitions.md
     - reference/cli-api.md

pyproject.toml

@@ -39,6 +39,9 @@ dependencies = [
     "cachetools",
     "structlog",
     "orjson",
+    "owlready2",
+    "fastapi",
+    "pygit2", # transitive from NeXusOntology submodule
 ]
 
 [project.urls]
@@ -107,6 +110,7 @@ nexus_schema = "pynxtools.nomad.entrypoints:nexus_schema"
 nexus_data_converter = "pynxtools.nomad.entrypoints:nexus_data_converter"
 nexus_app = "pynxtools.nomad.entrypoints:nexus_app"
 simple_nexus_example = "pynxtools.nomad.entrypoints:simple_nexus_example"
+ontology_service = "pynxtools.nomad.apis:ontology_service"
 
 [project.scripts]
 read_nexus = "pynxtools.nexus.nexus:main"
@@ -118,6 +122,7 @@ validate_nexus = "pynxtools.dataconverter.validate_file:validate_cli"
 pynxtools = ["definitions/**/*.xml", "definitions/**/*.xsd"]
 "pynxtools.dataconverter.readers.hall" = ["enum_map.json"]
 "pynxtools.dataconverter.readers.rii_database.formula_parser" = ["dispersion_function_grammar.lark"]
+"pynxtools.NeXusOntology.script" = ["*.py"]
 
 [tool.setuptools.packages.find]
 where = [
@@ -131,7 +136,10 @@ local_scheme = "node-and-date"
 
 [tool.ruff]
 include = ["src/*.py", "tests/*.py"]
-exclude = ["src/pynxtools/definitions"]
+exclude = [
+    "src/pynxtools/definitions",
+    "src/pynxtools/NeXusOntology",
+]
 line-length = 88
 indent-width = 4
 
@@ -178,9 +186,10 @@ ignore_missing_imports = true
 follow_imports = "silent"
 no_strict_optional = true
 disable_error_code = "import, annotation-unchecked"
-exclude = ["src/pynxtools/definitions/*"]
+exclude = ["src/pynxtools/definitions/*", "src/pynxtools/NeXusOntology/*"]
 
 [tool.uv]
 extra-index-url = [
 "https://gitlab.mpcdf.mpg.de/api/v4/projects/2187/packages/pypi/simple",
-]
+]
+