Skip to content

Commit d1a2bbb

Browse files
sanbrocksherjeelshabihPepe-Marquezlukaspie
authored
Example update (#559)
* accept open enumerations as no real enumerations * fix the type of ELN Entry; add reference to nexus result Entry; add ELN fields * Fixes IV_temp example to use eln_data.yaml * seprate overview card for the notes * adding nexus_view to NexusDataConverter * add an example for simple data converstion using external_eln_data.yaml; NexusDataconverter: if eln data comes from NOMAD, write it to <output>_eln_data.yaml * do not change input file list * split IV_temp example to 3 use cases * fix for bestfit; linting * allow converting multiple files as fiter is set * fix import * fix inputs * fix output * remove default value for filter * Added Plotly superclass to NeXusMeasurement and use it for APM. * similalry to all other base classes, convert NXobject also in NOMAD, so it becomes to Object * update metainfo schema test * fix tests * updating definitions to get dev_tools features * merging examples into subdirectories * fixing tests * linting * fixing tests for open enums * treat NX_CHAR_OR_NUMBER as NX_NUMBER * definitions update * ref log update --------- Co-authored-by: Sherjeel Shabih <shabihsherjeel@gmail.com> Co-authored-by: Pepe Marquez <pepemarquezprieto@gmail.com> Co-authored-by: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com>
1 parent aa77148 commit d1a2bbb

25 files changed

+1387
-594
lines changed

‎pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ nexus_parser = "pynxtools.nomad.entrypoints:nexus_parser"
100100
nexus_schema = "pynxtools.nomad.entrypoints:nexus_schema"
101101
nexus_data_converter = "pynxtools.nomad.entrypoints:nexus_data_converter"
102102
nexus_app = "pynxtools.nomad.entrypoints:nexus_app"
103-
iv_temp_example = "pynxtools.nomad.entrypoints:iv_temp_example"
103+
simple_nexus_example = "pynxtools.nomad.entrypoints:simple_nexus_example"
104104

105105
[project.scripts]
106106
read_nexus = "pynxtools.nexus.nexus:main"

‎src/pynxtools/dataconverter/readers/json_map/reader.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
from typing import Any, Tuple
2323

2424
import numpy as np
25+
import yaml
2526
import xarray
2627
from mergedeep import merge
2728

@@ -193,6 +194,9 @@ def read(
193194
elif file_extension == ".pickle":
194195
with open(file_path, "rb") as input_file: # type: ignore[assignment]
195196
data = pickle.load(input_file) # type: ignore[arg-type]
197+
elif file_extension == ".yaml":
198+
with open(file_path, "r") as input_file:
199+
merge(data, yaml.safe_load(input_file))
196200
else:
197201
is_hdf5 = False
198202
with open(file_path, "rb") as input_file:

‎src/pynxtools/definitions‎

Submodule definitions updated 668 files

‎src/pynxtools/nexus/nexus.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -357,6 +357,7 @@ def helper_get_inherited_nodes(hdf_info2, elist, pind, attr):
357357
elist[ind], hdf_node, hdf_name, hdf_class_name, act_nexus_type
358358
)
359359
if fit >= bestfit and newelem is not None:
360+
bestfit = fit
360361
html_name = get_node_name(newelem)
361362
return hdf_path, hdf_node, hdf_class_path, elist, pind, attr, html_name
362363

‎src/pynxtools/nomad/dataconverter.py‎

Lines changed: 94 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import os
12
import os.path
23
import re
34
from typing import Optional
@@ -6,7 +7,7 @@
67
import yaml
78

89
try:
9-
from nomad.datamodel.data import EntryData
10+
from nomad.datamodel.data import ArchiveSection, EntryData
1011
from nomad.metainfo import MEnum, Package, Quantity
1112
from nomad.units import ureg
1213
except ImportError as exc:
@@ -43,7 +44,14 @@ def transform(quantity_def, section, value, path):
4344
return value
4445

4546
def exclude(quantity_def, section):
46-
return quantity_def.name in ("reader", "input_files", "output", "nxdl")
47+
return quantity_def.name in (
48+
"reader",
49+
"nxdl",
50+
"input_files",
51+
"output",
52+
"filter",
53+
"nexus_view",
54+
)
4755

4856
eln_dict = archive.m_to_dict(transform=transform, exclude=exclude)
4957
del eln_dict["data"]["m_def"]
@@ -203,59 +211,98 @@ class NexusDataConverter(EntryData):
203211
default="output.nxs",
204212
)
205213

214+
filter = Quantity(
215+
type=str,
216+
description="Filter to select additional input files to be converted to NeXus",
217+
a_eln=dict(component="StringEditQuantity"),
218+
)
219+
220+
nexus_view = Quantity(
221+
type=ArchiveSection,
222+
description="Link to the NeXus Entry",
223+
)
224+
206225
def normalize(self, archive, logger):
207226
super(NexusDataConverter, self).normalize(archive, logger)
208227

209228
raw_path = archive.m_context.raw_path()
210229
eln_dict = create_eln_dict(archive)
211230

212-
if archive.data.input_files is None:
213-
archive.data.input_files = []
214-
231+
input_file_list = (
232+
list(archive.data.input_files) if archive.data.input_files else []
233+
)
215234
if len(eln_dict["data"]) > 0:
216-
write_yaml(archive, "eln_data.yaml", eln_dict)
217-
218-
if "eln_data.yaml" not in archive.data.input_files:
219-
archive.data.input_files.append("eln_data.yaml")
220-
221-
converter_params = {
222-
"reader": archive.data.reader,
223-
"nxdl": re.sub(".nxdl$", "", archive.data.nxdl),
224-
"input_file": [
225-
os.path.join(raw_path, file) for file in archive.data.input_files
226-
],
227-
"output": os.path.join(raw_path, archive.data.output),
228-
}
229-
# remove the nexus file and ensure that NOMAD knows that it is removed
230-
try:
231-
os.remove(os.path.join(raw_path, archive.data.output))
232-
archive.m_context.process_updated_raw_file(
233-
archive.data.output, allow_modify=True
234-
)
235-
except Exception as e:
236-
pass
237-
# create the new nexus file
238-
try:
239-
pynxtools_converter.logger = logger
240-
pynxtools_converter.helpers.logger = logger
241-
pynxtools_converter.convert(**converter_params)
242-
except Exception as e:
243-
logger.error(
244-
"could not convert to nxs", mainfile=archive.data.output, exc_info=e
245-
)
246-
raise e
247-
# parse the new nexus file
248-
try:
249-
archive.m_context.process_updated_raw_file(
250-
archive.data.output, allow_modify=True
251-
)
252-
except Exception as e:
253-
logger.error(
254-
"could not trigger processing", mainfile=archive.data.output, exc_info=e
255-
)
256-
raise e
235+
eln_fname = f"{archive.data.output}_eln_data.yaml"
236+
write_yaml(archive, eln_fname, eln_dict)
237+
input_file_list.append(eln_fname)
238+
239+
# collect extra input files
240+
input_list = [os.path.join(raw_path, file) for file in input_file_list]
241+
if self.filter:
242+
try:
243+
extra_inputs = [
244+
f for f in os.listdir(raw_path) if re.match(self.filter, f)
245+
]
246+
except Exception as e:
247+
logger.error(
248+
"could not get file list accordonf to the filter provided",
249+
mainfile=archive.data.output,
250+
exc_info=e,
251+
)
252+
extra_inputs = []
257253
else:
258-
logger.info("triggered processing", mainfile=archive.data.output)
254+
extra_inputs = []
255+
extra_inputs += [""]
256+
# convert all files
257+
for extra_input in extra_inputs:
258+
if len(extra_input) > 0:
259+
input = input_list + [os.path.join(raw_path, extra_input)]
260+
output = f"{extra_input.replace('.', '_')}.nxs"
261+
else:
262+
input = input_list
263+
output = archive.data.output
264+
converter_params = {
265+
"reader": archive.data.reader,
266+
"nxdl": re.sub(".nxdl$", "", archive.data.nxdl),
267+
"input_file": input,
268+
"output": os.path.join(raw_path, output),
269+
}
270+
# remove the nexus file and ensure that NOMAD knows that it is removed
271+
try:
272+
os.remove(os.path.join(raw_path, output))
273+
archive.m_context.process_updated_raw_file(
274+
archive.data.output, allow_modify=True
275+
)
276+
except Exception as e:
277+
pass
278+
# create the new nexus file
279+
try:
280+
pynxtools_converter.logger = logger
281+
pynxtools_converter.helpers.logger = logger
282+
pynxtools_converter.convert(**converter_params)
283+
except Exception as e:
284+
logger.error("could not convert to nxs", mainfile=output, exc_info=e)
285+
continue
286+
# parse the new nexus file
287+
try:
288+
archive.m_context.process_updated_raw_file(output, allow_modify=True)
289+
except Exception as e:
290+
logger.error(
291+
"could not trigger processing", mainfile=output, exc_info=e
292+
)
293+
continue
294+
else:
295+
logger.info("triggered processing", mainfile=output)
296+
# reference the generated nexus file
297+
try:
298+
self.nexus_view = f"../upload/archive/mainfile/{output}#/data"
299+
except Exception as e:
300+
logger.error(
301+
"could not reference the generate nexus file",
302+
mainfile=output,
303+
exc_info=e,
304+
)
305+
continue
259306

260307

261308
m_package.__init_metainfo__()

‎src/pynxtools/nomad/entrypoints.py‎

Lines changed: 22 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -353,13 +353,29 @@ def load(self):
353353
),
354354
)
355355

356-
iv_temp_example = ExampleUploadEntryPoint(
357-
title="Sensor Scan - IV Temperature Curve",
358-
category="FAIRmat examples",
356+
simple_nexus_example = ExampleUploadEntryPoint(
357+
title="Simple NeXus Example",
358+
category="NeXus Experiment Examples",
359359
description="""
360-
This example shows users how to take data from a Python framework and map it out to a Nexus application definition for IV Temperature measurements, [`NXiv_temp`](https://fairmat-nfdi.github.io/nexus_definitions/classes/contributed_definitions/NXiv_temp.html).
361-
We use the Nexus ELN features of NOMAD to generate a Nexus file.
360+
This example show 3 use cases on how NeXus experiment data can be handled in NOMAD.
361+
Example 1 - ELN Export
362+
This example shows how a simple ELN can be set up in NOMAD which can be then
363+
exported in to an RDM agnostic eln_data.yaml format. The example also shows how such
364+
eln file can be used together with some experiment data to be converted by pynxtools
365+
to a valid NeXus file.
366+
Example 2 - Interface for Data Conversion to NeXus Format
367+
This example shows how NOMAD GUI allows converting experiment data with
368+
attached eln notes to NeXus file.
369+
Example 3 - Sensor Scan - IV Temperature Curve
370+
This example shows how experimental data can be mapped to a Nexus application definition.
371+
Here, data from an IV Temperature measurements as taken by a Python framework is
372+
converted to [`NXiv_temp`](https://fairmat-nfdi.github.io/nexus_definitions/classes/contributed_definitions/NXiv_temp.html).
373+
We also demonstrate the use of Nexus ELN features of NOMAD to add further details
374+
which were not provided by the data acquisition software.
375+
This example combines Example 1, and 2, and demonstrates how a NOMAD ELN can be built
376+
to collect additional information, and combine it with experimental data to convert
377+
them into exportable NeXus file, which is also directly searchable in NOMAD.
362378
""",
363379
plugin_package="pynxtools",
364-
resources=["nomad/examples/iv_temp/*"],
380+
resources=["nomad/examples/*"],
365381
)

src/pynxtools/nomad/examples/iv_temp/IV_temp.mapping.json renamed to src/pynxtools/nomad/examples/IV_temp.mapping.json

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,8 @@
4141
"/ENTRY[entry]/definition/@version": "1",
4242
"/ENTRY[entry]/experiment_identifier": "/metadata_start/uid",
4343
"/ENTRY[entry]/experiment_description": "A simple IV temperature experiment.",
44-
"/ENTRY[entry]/start_time": "/metadata_start/time"
44+
"/ENTRY[entry]/start_time": "/metadata_start/time",
45+
"/ENTRY[entry]/NOTE[note]/author": "/notes/author",
46+
"/ENTRY[entry]/NOTE[note]/description": "/notes/note"
4547

4648
}

src/pynxtools/nomad/examples/iv_temp/README.md renamed to src/pynxtools/nomad/examples/README.md

File renamed without changes.

‎src/pynxtools/nomad/examples/eln_export/eln_export.archive.json‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
{
2+
"data": {
3+
"m_def": "../upload/raw/eln_export/eln_export.schema.archive.yaml#/definitions/section_definitions/0",
4+
"output": "eln_export/eln_data.yaml",
5+
"notes": {
6+
"author": "name",
7+
"note": "example note"
8+
}
9+
},
10+
"m_ref_archives": {}
11+
}

‎src/pynxtools/nomad/examples/eln_export/eln_export.ipynb‎

Lines changed: 196 additions & 0 deletions
Large diffs are not rendered by default.

0 commit comments

Comments
 (0)