Skip to content

Commit 8cc3e91

Browse files
authored
Merge pull request #578 from FAIRmat-NFDI/update-defs-and-related-tools
update definitions and change tools related to new NIAC ideas
2 parents b49cb08 + b454916 commit 8cc3e91

File tree

19 files changed

+1287
-1177
lines changed

19 files changed

+1287
-1177
lines changed

.github/workflows/plugin_test.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,10 +30,10 @@ jobs:
3030
branch: main
3131
tests_to_run: tests/.
3232
- plugin: pynxtools-igor
33-
branch: update-definitions
33+
branch: main
3434
tests_to_run: tests/.
3535
- plugin: pynxtools-mpes
36-
branch: update-definitions
36+
branch: main
3737
tests_to_run: tests/.
3838
- plugin: pynxtools-raman
3939
branch: main

CITATION.cff

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ message:
44
If you use this software, please cite it using the
55
metadata from this file.
66
type: software
7-
version: 0.9.3
7+
version: 0.10.0
88
authors:
99
- given-names: Sherjeel
1010
family-names: Shabih

pyproject.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ dependencies = [
3636
"lxml>=4.9.1",
3737
"toposort>=1.10.0",
3838
"anytree",
39+
"pint==0.17",
3940
]
4041

4142
[project.urls]

src/pynxtools/data/NXtest.nxdl.xml

Lines changed: 38 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -19,16 +19,37 @@
1919
<item value="NXtest"/>
2020
</enumeration>
2121
</field>
22-
<group type="NXdata" name="OPTIONAL_group" optional="true">
22+
<group type="NXdata" name="OPTIONAL_group" optional="true" nameType="partial">
2323
<field name="required_field" required="true" type="NX_INT">
2424
<doc>A dummy entry to test optional parent check for a required child.</doc>
2525
</field>
2626
<field name="optional_field" optional="true" type="NX_INT">
2727
<doc>A dummy entry to test optional parent check for an optional child.</doc>
2828
</field>
2929
</group>
30-
<group type="NXdata" name="NXODD_name">
31-
<field name="anamethatRENAMES" nameType="any" type="NX_INT" units="NX_UNITLESS"/>
30+
<group type="NXdata" name="specified_group_with_no_name_type">
31+
<doc>A group with a (specified) name, but nameType not given explicitly.</doc>
32+
<field name="specified_field_with_no_name_type" type="NX_FLOAT" units="NX_ANY">
33+
<attribute name="specified_attr_in_field_with_no_name_type"/>
34+
</field>
35+
<attribute name="specified_attr_with_no_name_type"/>
36+
</group>
37+
<group type="NXdata" name="specified_group" nameType="specified">
38+
<doc>A group with a name and nameType="specified".</doc>
39+
<field name="specified_field" type="NX_FLOAT" units="NX_ANY">
40+
<attribute name="specified_attr_in_field"/>
41+
</field>
42+
<attribute name="specified_attr"/>
43+
</group>
44+
<group type="NXdata" name="any_groupGROUP" nameType="any">
45+
<doc>A group with a name and nameType="any".</doc>
46+
<field name="any_fieldFIELD" nameType="any" type="NX_FLOAT" units="NX_ANY">
47+
<attribute name="any_attrATTR_in_field" nameType="any"/>
48+
</field>
49+
<attribute name="any_attrATTR" nameType="any"/>
50+
</group>
51+
<group type="NXdata" name="NXODD_name" nameType="partial">
52+
<field name="anamethatRENAMES" nameType="partial" type="NX_INT" units="NX_UNITLESS"/>
3253
<field name="float_value" type="NX_FLOAT" optional="true" units="NX_ENERGY">
3354
<doc>A dummy entry for a float value.</doc>
3455
</field>
@@ -52,10 +73,10 @@
5273
</field>
5374
<field name="type">
5475
<enumeration>
55-
<item value="1st type" />
56-
<item value="2nd type" />
57-
<item value="3rd type" />
58-
<item value="4th type" />
76+
<item value="1st type"/>
77+
<item value="2nd type"/>
78+
<item value="3rd type"/>
79+
<item value="4th type"/>
5980
</enumeration>
6081
<attribute name="array" type="NX_INT">
6182
<enumeration>
@@ -64,6 +85,12 @@
6485
</enumeration>
6586
</attribute>
6687
</field>
88+
<field name="type2" optional="true">
89+
<enumeration open="true">
90+
<item value="1st type open"/>
91+
<item value="2nd type open"/>
92+
</enumeration>
93+
</field>
6794
<attribute name="group_attribute">
6895
</attribute>
6996
<attribute name="signal">
@@ -94,5 +121,9 @@
94121
<doc>A required NXuser entry.</doc>
95122
</field>
96123
</group>
124+
<group name="identified_calibration" type="NXcalibration" optional="true">
125+
<field name="identifier_1"/>
126+
</group>
127+
<group name="named_collection" type="NXcollection" optional="true"/>
97128
</group>
98129
</definition>

src/pynxtools/dataconverter/helpers.py

Lines changed: 55 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -48,24 +48,25 @@
4848
class ValidationProblem(Enum):
4949
UnitWithoutDocumentation = 1
5050
InvalidEnum = 2
51-
MissingRequiredGroup = 3
52-
MissingRequiredField = 4
53-
MissingRequiredAttribute = 5
54-
InvalidType = 6
55-
InvalidDatetime = 7
56-
IsNotPosInt = 8
57-
ExpectedGroup = 9
58-
MissingDocumentation = 10
59-
MissingUnit = 11
60-
ChoiceValidationError = 12
61-
UnitWithoutField = 13
62-
AttributeForNonExistingField = 14
63-
BrokenLink = 15
64-
FailedNamefitting = 16
65-
NXdataMissingSignalData = 17
66-
NXdataMissingAxisData = 18
67-
NXdataAxisMismatch = 19
68-
KeyToBeRemoved = 20
51+
OpenEnumWithNewItem = 3
52+
MissingRequiredGroup = 4
53+
MissingRequiredField = 5
54+
MissingRequiredAttribute = 6
55+
InvalidType = 7
56+
InvalidDatetime = 8
57+
IsNotPosInt = 9
58+
ExpectedGroup = 10
59+
MissingDocumentation = 11
60+
MissingUnit = 12
61+
ChoiceValidationError = 13
62+
UnitWithoutField = 14
63+
AttributeForNonExistingField = 15
64+
BrokenLink = 16
65+
FailedNamefitting = 17
66+
NXdataMissingSignalData = 18
67+
NXdataMissingAxisData = 19
68+
NXdataAxisMismatch = 20
69+
KeyToBeRemoved = 21
6970

7071

7172
class Collector:
@@ -85,7 +86,11 @@ def _log(self, path: str, log_type: ValidationProblem, value: Optional[Any], *ar
8586
)
8687
elif log_type == ValidationProblem.InvalidEnum:
8788
logger.warning(
88-
f"The value at {path} should be one of the following: {value}"
89+
f"The value at {path} should be one of the following: {value}."
90+
)
91+
elif log_type == ValidationProblem.OpenEnumWithNewItem:
92+
logger.info(
93+
f"The value at {path} does not match with the enumerated items from the open enumeration: {value}."
8994
)
9095
elif log_type == ValidationProblem.MissingRequiredGroup:
9196
logger.warning(f"The required group, {path}, hasn't been supplied.")
@@ -164,7 +169,10 @@ def collect_and_log(
164169
if self.logging and path + str(log_type) + str(value) not in self.data:
165170
self._log(path, log_type, value, *args, **kwargs)
166171
# info messages should not fail validation
167-
if log_type not in (ValidationProblem.UnitWithoutDocumentation,):
172+
if log_type not in (
173+
ValidationProblem.UnitWithoutDocumentation,
174+
ValidationProblem.OpenEnumWithNewItem,
175+
):
168176
self.data.add(path + str(log_type) + str(value))
169177

170178
def has_validation_problems(self):
@@ -266,7 +274,7 @@ def get_all_parents_for(xml_elem: ET._Element) -> List[ET._Element]:
266274
root = get_appdef_root(xml_elem)
267275
inheritance_chain = []
268276
extends = root.get("extends")
269-
while extends is not None and extends != "NXobject":
277+
while extends is not None:
270278
parent_xml_root, _ = get_nxdl_root_and_path(extends)
271279
extends = parent_xml_root.get("extends")
272280
inheritance_chain.append(parent_xml_root)
@@ -489,6 +497,15 @@ def contains_uppercase(field_name: Optional[str]) -> bool:
489497
return any(char.isupper() for char in field_name)
490498

491499

500+
def is_variadic(name: str, name_type: str) -> bool:
501+
"""
502+
Determine if a name is variadic based on its nameType.
503+
"""
504+
if name:
505+
return False if name_type == "specified" else True
506+
return True
507+
508+
492509
def convert_nexus_to_suggested_name(nexus_name):
493510
"""Helper function to suggest a name for a group from its NeXus class."""
494511
if contains_uppercase(nexus_name):
@@ -646,7 +663,9 @@ def convert_str_to_bool_safe(value: str) -> Optional[bool]:
646663
raise ValueError(f"Could not interpret string '{value}' as boolean.")
647664

648665

649-
def is_valid_data_field(value: Any, nxdl_type: str, nxdl_enum: list, path: str) -> Any:
666+
def is_valid_data_field(
667+
value: Any, nxdl_type: str, nxdl_enum: list, nxdl_enum_open: bool, path: str
668+
) -> Any:
650669
# todo: Check this function and write test for it. It seems the function is not
651670
# working as expected.
652671
"""Checks whether a given value is valid according to the type defined in the NXDL.
@@ -688,11 +707,18 @@ def is_valid_data_field(value: Any, nxdl_type: str, nxdl_enum: list, path: str)
688707

689708
# Check enumeration
690709
if nxdl_enum is not None and value not in nxdl_enum:
691-
collector.collect_and_log(
692-
path,
693-
ValidationProblem.InvalidEnum,
694-
nxdl_enum,
695-
)
710+
if nxdl_enum_open:
711+
collector.collect_and_log(
712+
path,
713+
ValidationProblem.OpenEnumWithNewItem,
714+
nxdl_enum,
715+
)
716+
else:
717+
collector.collect_and_log(
718+
path,
719+
ValidationProblem.InvalidEnum,
720+
nxdl_enum,
721+
)
696722

697723
return value
698724

@@ -871,8 +897,8 @@ def update_and_warn(key: str, value: str):
871897
"https://github.com/FAIRmat-NFDI/nexus_definitions/"
872898
f"blob/{get_nexus_version_hash()}",
873899
)
874-
update_and_warn("/@NeXus_version", get_nexus_version())
875-
update_and_warn("/@HDF5_version", ".".join(map(str, h5py.h5.get_libversion())))
900+
update_and_warn("/@NeXus_release", get_nexus_version())
901+
update_and_warn("/@HDF5_Version", ".".join(map(str, h5py.h5.get_libversion())))
876902
update_and_warn("/@h5py_version", h5py.__version__)
877903

878904

0 commit comments

Comments
 (0)