diff --git a/.gitignore b/.gitignore index 28514e8d9..fe9b6f9c4 100644 --- a/.gitignore +++ b/.gitignore @@ -21,7 +21,7 @@ htmlcov/ docs/build/ .hypothesis/ -# customized config files +# Customized config files sdk/test/test_config.ini # Schema files needed for testing sdk/test/adapter/schemas @@ -31,5 +31,6 @@ sdk/basyx/version.py compliance_tool/aas_compliance_tool/version.py server/app/version.py -# ignore the content of the server storage +# Ignore the content of the server storage +server/input/ server/storage/ diff --git a/sdk/basyx/aas/adapter/__init__.py b/sdk/basyx/aas/adapter/__init__.py index 7f96702e9..0fca01291 100644 --- a/sdk/basyx/aas/adapter/__init__.py +++ b/sdk/basyx/aas/adapter/__init__.py @@ -7,3 +7,45 @@ Python SDK objects to/from XML. * :ref:`aasx `: This package offers functions for reading and writing AASX-files. """ + +from basyx.aas.adapter.aasx import AASXReader, DictSupplementaryFileContainer +from basyx.aas.adapter.json import read_aas_json_file_into +from basyx.aas.adapter.xml import read_aas_xml_file_into +from basyx.aas.model.provider import DictObjectStore +from pathlib import Path +from typing import Union + + +def load_directory(directory: Union[Path, str]) -> tuple[DictObjectStore, DictSupplementaryFileContainer]: + """ + Create a new :class:`~basyx.aas.model.provider.DictObjectStore` and use it to load Asset Administration Shell and + Submodel files in ``AASX``, ``JSON`` and ``XML`` format from a given directory into memory. Additionally, load all + embedded supplementary files into a new :class:`~basyx.aas.adapter.aasx.DictSupplementaryFileContainer`. + + :param directory: :class:`~pathlib.Path` or ``str`` pointing to the directory containing all Asset Administration + Shell and Submodel files to load + :return: Tuple consisting of a :class:`~basyx.aas.model.provider.DictObjectStore` and a + :class:`~basyx.aas.adapter.aasx.DictSupplementaryFileContainer` containing all loaded data + """ + + dict_object_store: DictObjectStore = DictObjectStore() + file_container: DictSupplementaryFileContainer = DictSupplementaryFileContainer() + + directory = Path(directory) + + for file in directory.iterdir(): + if not file.is_file(): + continue + + suffix = file.suffix.lower() + if suffix == ".json": + with open(file) as f: + read_aas_json_file_into(dict_object_store, f) + elif suffix == ".xml": + with open(file) as f: + read_aas_xml_file_into(dict_object_store, f) + elif suffix == ".aasx": + with AASXReader(file) as reader: + reader.read_into(object_store=dict_object_store, file_store=file_container) + + return dict_object_store, file_container diff --git a/sdk/basyx/aas/adapter/aasx.py b/sdk/basyx/aas/adapter/aasx.py index a63dca6c8..8bb5958f6 100644 --- a/sdk/basyx/aas/adapter/aasx.py +++ b/sdk/basyx/aas/adapter/aasx.py @@ -871,3 +871,6 @@ def __contains__(self, item: object) -> bool: def __iter__(self) -> Iterator[str]: return iter(self._name_map) + + def __len__(self) -> int: + return len(self._name_map) diff --git a/sdk/basyx/aas/adapter/json/json_serialization.py b/sdk/basyx/aas/adapter/json/json_serialization.py index 024226d97..0b0df0164 100644 --- a/sdk/basyx/aas/adapter/json/json_serialization.py +++ b/sdk/basyx/aas/adapter/json/json_serialization.py @@ -136,7 +136,7 @@ def _abstract_classes_to_json(cls, obj: object) -> Dict[str, object]: if obj.description: data['description'] = obj.description try: - ref_type = next(iter(t for t in inspect.getmro(type(obj)) if t in model.KEY_TYPES_CLASSES)) + ref_type = model.resolve_referable_class_in_key_types(obj) except StopIteration as e: raise TypeError("Object of type {} is Referable but does not inherit from a known AAS type" .format(obj.__class__.__name__)) from e diff --git a/sdk/basyx/aas/model/__init__.py b/sdk/basyx/aas/model/__init__.py index e541968b5..2ddd93b51 100644 --- a/sdk/basyx/aas/model/__init__.py +++ b/sdk/basyx/aas/model/__init__.py @@ -39,3 +39,17 @@ RelationshipElement: KeyTypes.RELATIONSHIP_ELEMENT, SubmodelElement: KeyTypes.SUBMODEL_ELEMENT, # type: ignore } + + +def resolve_referable_class_in_key_types(referable: Referable) -> type: + """ + Returns the type of referable if the type is given in `KEY_TYPES_CLASSES`, otherwise return the first parent class + in inheritance chain of the referable which is given in `KEY_TYPES_CLASSES`. + + :raises TypeError: If the type of the referable or any of its parent classes is not given in `KEY_TYPES_CLASSES`. + """ + try: + ref_type = next(iter(t for t in inspect.getmro(type(referable)) if t in KEY_TYPES_CLASSES)) + except StopIteration: + raise TypeError(f"Could not find a matching class in KEY_TYPES_CLASSES for {type(referable)}") + return ref_type diff --git a/sdk/basyx/aas/model/base.py b/sdk/basyx/aas/model/base.py index 60b6d43fb..35ccad5a1 100644 --- a/sdk/basyx/aas/model/base.py +++ b/sdk/basyx/aas/model/base.py @@ -41,6 +41,8 @@ VersionType = str ValueTypeIEC61360 = str +MAX_RECURSION_DEPTH = 32*2 # see https://github.com/admin-shell-io/aas-specs-metamodel/issues/333 + @unique class KeyTypes(Enum): @@ -453,25 +455,31 @@ def from_referable(referable: "Referable") -> "Key": """ # Get the `type` by finding the first class from the base classes list (via inspect.getmro), that is contained # in KEY_ELEMENTS_CLASSES - from . import KEY_TYPES_CLASSES, SubmodelElementList - try: - key_type = next(iter(KEY_TYPES_CLASSES[t] - for t in inspect.getmro(type(referable)) - if t in KEY_TYPES_CLASSES)) - except StopIteration: - key_type = KeyTypes.PROPERTY + key_type = Key._get_key_type_for_referable(referable) + key_value = Key._get_key_value_for_referable(referable) + return Key(key_type, key_value) + @staticmethod + def _get_key_type_for_referable(referable: "Referable") -> KeyTypes: + from . import KEY_TYPES_CLASSES, resolve_referable_class_in_key_types + ref_type = resolve_referable_class_in_key_types(referable) + key_type = KEY_TYPES_CLASSES[ref_type] + return key_type + + @staticmethod + def _get_key_value_for_referable(referable: "Referable") -> str: + from . import SubmodelElementList if isinstance(referable, Identifiable): - return Key(key_type, referable.id) + return referable.id elif isinstance(referable.parent, SubmodelElementList): try: - return Key(key_type, str(referable.parent.value.index(referable))) # type: ignore + return str(referable.parent.value.index(referable)) # type: ignore except ValueError as e: raise ValueError(f"Object {referable!r} is not contained within its parent {referable.parent!r}") from e else: if referable.id_short is None: - raise ValueError(f"Can't create Key for {referable!r} without an id_short!") - return Key(key_type, referable.id_short) + raise ValueError(f"Can't create Key value for {referable!r} without an id_short!") + return referable.id_short _NSO = TypeVar('_NSO', bound=Union["Referable", "Qualifier", "HasSemantics", "Extension"]) @@ -614,26 +622,75 @@ def __init__(self): self.parent: Optional[UniqueIdShortNamespace] = None def __repr__(self) -> str: - reversed_path = [] + root = self.get_identifiable_root() + try: + id_short_path = self.get_id_short_path() + except (ValueError, AttributeError): + id_short_path = self.id_short if self.id_short is not None else "" + item_cls_name = self.__class__.__name__ + + if root is None: + item_path = f"[{id_short_path}]" if id_short_path else "" + else: + item_path = f"[{root.id} / {id_short_path}]" if id_short_path else f"[{root.id}]" + + return f"{item_cls_name}{item_path}" + + def get_identifiable_root(self) -> Optional["Identifiable"]: + """ + Get the root :class:`~.Identifiable` of this referable, if it exists. + + :return: The root :class:`~.Identifiable` or None if no such root exists + """ item = self # type: Any - if item.id_short is not None: - from .submodel import SubmodelElementList - while item is not None: - if isinstance(item, Identifiable): - reversed_path.append(item.id) - break - elif isinstance(item, Referable): - if isinstance(item.parent, SubmodelElementList): - reversed_path.append(f"{item.parent.id_short}[{item.parent.value.index(item)}]") - item = item.parent - else: - reversed_path.append(item.id_short) - item = item.parent - else: - raise AttributeError('Referable must have an identifiable as root object and only parents that are ' - 'referable') + while item is not None: + if isinstance(item, Identifiable): + return item + elif isinstance(item, Referable): + item = item.parent + else: + raise AttributeError('Referable must have an identifiable as root object and only parents that are ' + 'referable') + return None - return self.__class__.__name__ + ("[{}]".format(" / ".join(reversed(reversed_path))) if reversed_path else "") + def get_id_short_path(self) -> str: + """ + Get the id_short path of this referable, i.e. the id_short of this referable and all its parents. + + :return: The id_short path as a string, e.g. "MySECollection.MySEList[2]MySubProperty1" + """ + path_list = self.get_id_short_path_as_a_list() + return self.build_id_short_path(path_list) + + def get_id_short_path_as_a_list(self) -> List[str]: + """ + Get the id_short path of this referable as a list of id_shorts and indexes. + + :return: The id_short path as a list, e.g. '["MySECollection", "MySEList", "2", "MySubProperty1"]' + :raises ValueError: If this referable has no id_short or + if its parent is not a :class:`~basyx.aas.model.submodel.SubmodelElementList` + :raises AttributeError: If the parent chain is broken, i.e. if a parent is neither a :class:`~.Referable` nor an + :class:`~.Identifiable` + """ + from .submodel import SubmodelElementList + if self.id_short is None and not isinstance(self.parent, SubmodelElementList): + raise ValueError(f"Can't create id_short_path for {self.__class__.__name__} without an id_short or " + f"if its parent is a SubmodelElementList!") + + item = self # type: Any + path: List[str] = [] + while item is not None: + if not isinstance(item, Referable): + raise AttributeError('Referable must have an identifiable as root object and only parents that are ' + 'referable') + if isinstance(item, Identifiable): + break + elif isinstance(item.parent, SubmodelElementList): + path.insert(0, str(item.parent.value.index(item))) + else: + path.insert(0, item.id_short) + item = item.parent + return path def _get_id_short(self) -> Optional[NameType]: return self._id_short @@ -653,6 +710,49 @@ def _set_category(self, category: Optional[NameType]): def _get_category(self) -> Optional[NameType]: return self._category + @classmethod + def parse_id_short_path(cls, id_short_path: str) -> List[str]: + """ + Parse an id_short_path string into a list of id_shorts and indexes. + + :param id_short_path: The id_short_path string, e.g. "MySECollection.MySEList[2]MySubProperty1" + :return: The id_short path as a list, e.g. '["MySECollection", "MySEList", "2", "MySubProperty1"]' + """ + id_shorts_and_indexes = [] + for part in id_short_path.split("."): + id_short = part[0:part.find('[')] if '[' in part else part + id_shorts_and_indexes.append(id_short) + + indexes_part = part.removeprefix(id_short) + if indexes_part: + if not re.fullmatch(r'(?:\[\d+\])+', indexes_part): + raise ValueError(f"Invalid index format in id_short_path: '{id_short_path}', part: '{part}'") + indexes = indexes_part.strip("[]").split("][") + id_shorts_and_indexes.extend(indexes) + cls.validate_id_short_path(id_shorts_and_indexes) + return id_shorts_and_indexes + + @classmethod + def build_id_short_path(cls, id_short_path: Iterable[str]) -> str: + """ + Build an id_short_path string from a list of id_shorts and indexes. + """ + if isinstance(id_short_path, str): + raise ValueError("id_short_path must be an Iterable of strings, not a single string") + path_list_with_dots_and_brackets = [f"[{part}]" if part.isdigit() else f".{part}" for part in id_short_path] + id_short_path = "".join(path_list_with_dots_and_brackets).removeprefix(".") + return id_short_path + + @classmethod + def validate_id_short_path(cls, id_short_path: Union[str, NameType, Iterable[NameType]]): + if isinstance(id_short_path, str): + id_short_path = cls.parse_id_short_path(id_short_path) + for id_short in id_short_path: + if id_short.isdigit(): + # This is an index, skip validation + continue + cls.validate_id_short(id_short) + @classmethod def validate_id_short(cls, id_short: NameType) -> None: """ @@ -1001,22 +1101,24 @@ def from_referable(referable: Referable) -> "ModelReference": object's ancestors """ # Get the first class from the base classes list (via inspect.getmro), that is contained in KEY_ELEMENTS_CLASSES - from . import KEY_TYPES_CLASSES + from . import resolve_referable_class_in_key_types try: - ref_type = next(iter(t for t in inspect.getmro(type(referable)) if t in KEY_TYPES_CLASSES)) + ref_type = resolve_referable_class_in_key_types(referable) except StopIteration: ref_type = Referable ref: Referable = referable keys: List[Key] = [] while True: - keys.append(Key.from_referable(ref)) + keys.insert(0, Key.from_referable(ref)) if isinstance(ref, Identifiable): - keys.reverse() return ModelReference(tuple(keys), ref_type) if ref.parent is None or not isinstance(ref.parent, Referable): - raise ValueError("The given Referable object is not embedded within an Identifiable object") + raise ValueError(f"The given Referable object is not embedded within an Identifiable object: {ref}") ref = ref.parent + if len(keys) > MAX_RECURSION_DEPTH: + raise ValueError(f"The given Referable object is embedded in >64 layers of Referables " + f"or there is a loop in the parent chain {ref}") @_string_constraints.constrain_content_type("content_type") @@ -1624,12 +1726,12 @@ def __init__(self) -> None: super().__init__() self.namespace_element_sets: List[NamespaceSet] = [] - def get_referable(self, id_short: Union[NameType, Iterable[NameType]]) -> Referable: + def get_referable(self, id_short_path: Union[str, NameType, Iterable[NameType]]) -> Referable: """ Find a :class:`~.Referable` in this Namespace by its id_short or by its id_short path. The id_short path may contain :class:`~basyx.aas.model.submodel.SubmodelElementList` indices. - :param id_short: id_short or id_short path as any :class:`Iterable` + :param id_short_path: id_short or id_short path as a str or any :class:`Iterable` :returns: :class:`~.Referable` :raises TypeError: If one of the intermediate objects on the path is not a :class:`~.UniqueIdShortNamespace` @@ -1638,10 +1740,10 @@ def get_referable(self, id_short: Union[NameType, Iterable[NameType]]) -> Refera :raises KeyError: If no such :class:`~.Referable` can be found """ from .submodel import SubmodelElementList - if isinstance(id_short, NameType): - id_short = [id_short] + if isinstance(id_short_path, (str, NameType)): + id_short_path = Referable.parse_id_short_path(id_short_path) item: Union[UniqueIdShortNamespace, Referable] = self - for id_ in id_short: + for id_ in id_short_path: # This is redundant on first iteration, but it's a negligible overhead. # Also, ModelReference.resolve() relies on this check. if not isinstance(item, UniqueIdShortNamespace): diff --git a/sdk/basyx/aas/model/provider.py b/sdk/basyx/aas/model/provider.py index b7acea8b8..d13758308 100644 --- a/sdk/basyx/aas/model/provider.py +++ b/sdk/basyx/aas/model/provider.py @@ -11,7 +11,7 @@ """ import abc -from typing import MutableSet, Iterator, Generic, TypeVar, Dict, List, Optional, Iterable, Set +from typing import MutableSet, Iterator, Generic, TypeVar, Dict, List, Optional, Iterable, Set, Tuple, cast from .base import Identifier, Identifiable @@ -67,7 +67,7 @@ class AbstractObjectStore(AbstractObjectProvider, MutableSet[_IT], Generic[_IT], :class:`~basyx.aas.model.base.Identifier` – allow to add and delete objects (i.e. behave like a Python set). This includes local object stores (like :class:`~.DictObjectStore`) and specific object stores (like :class:`~basyx.aas.backend.couchdb.CouchDBObjectStore` and - :class `~basyx.aas.backend.local_file.LocalFileObjectStore`). + :class:`~basyx.aas.backend.local_file.LocalFileObjectStore`). The AbstractObjectStore inherits from the :class:`~collections.abc.MutableSet` abstract collections class and therefore implements all the functions of this class. @@ -80,6 +80,36 @@ def update(self, other: Iterable[_IT]) -> None: for x in other: self.add(x) + def sync(self, other: Iterable[_IT], overwrite: bool) -> Tuple[int, int, int]: + """ + Merge :class:`Identifiables ` from an + :class:`~collections.abc.Iterable` into this :class:`~basyx.aas.model.provider.AbstractObjectStore`. + + :param other: :class:`~collections.abc.Iterable` to sync with + :param overwrite: Flag to overwrite existing :class:`Identifiables ` in this + :class:`~basyx.aas.model.provider.AbstractObjectStore` with updated versions from ``other``, + :class:`Identifiables ` unique to this + :class:`~basyx.aas.model.provider.AbstractObjectStore` are always preserved + :return: Counts of processed :class:`Identifiables ` as + ``(added, overwritten, skipped)`` + """ + + added, overwritten, skipped = 0, 0, 0 + for identifiable in other: + identifiable_id = identifiable.id + if identifiable_id in self: + if overwrite: + existing = self.get_identifiable(identifiable_id) + self.discard(cast(_IT, existing)) + self.add(identifiable) + overwritten += 1 + else: + skipped += 1 + else: + self.add(identifiable) + added += 1 + return added, overwritten, skipped + class DictObjectStore(AbstractObjectStore[_IT], Generic[_IT]): """ diff --git a/sdk/pyproject.toml b/sdk/pyproject.toml index 453b9e189..588d18407 100644 --- a/sdk/pyproject.toml +++ b/sdk/pyproject.toml @@ -36,9 +36,9 @@ classifiers = [ ] requires-python = ">=3.9" dependencies = [ - "lxml>=5.3", + "lxml>=6.0.2", "python-dateutil>=2.8,<3", - "pyecma376-2>=1.0.1" + "pyecma376-2>=1.0.2" ] [project.optional-dependencies] diff --git a/sdk/test/examples/test_helpers.py b/sdk/test/examples/test_helpers.py index faca8602b..0257b8bca 100644 --- a/sdk/test/examples/test_helpers.py +++ b/sdk/test/examples/test_helpers.py @@ -227,7 +227,7 @@ def test_submodel_element_collection_checker(self): self.assertEqual("FAIL: Attribute value of SubmodelElementCollection[Collection] must contain 2 " "SubmodelElements (count=1)", repr(next(checker_iterator))) - self.assertEqual("FAIL: Submodel Element Property[Collection / Prop1] must exist ()", + self.assertEqual("FAIL: Submodel Element Property[Collection.Prop1] must exist ()", repr(next(checker_iterator))) collection.add_referable(property) @@ -291,7 +291,7 @@ def test_annotated_relationship_element(self): self.assertEqual("FAIL: Attribute annotation of AnnotatedRelationshipElement[test] must contain 1 DataElements " "(count=0)", repr(next(checker_iterator))) - self.assertEqual("FAIL: Annotation Property[test / ExampleAnnotatedProperty] must exist ()", + self.assertEqual("FAIL: Annotation Property[test.ExampleAnnotatedProperty] must exist ()", repr(next(checker_iterator))) def test_submodel_checker(self): diff --git a/sdk/test/model/test_base.py b/sdk/test/model/test_base.py index 836980025..460bce563 100644 --- a/sdk/test/model/test_base.py +++ b/sdk/test/model/test_base.py @@ -41,7 +41,7 @@ def test_from_referable(self): self.assertEqual(model.Key(model.KeyTypes.MULTI_LANGUAGE_PROPERTY, "0"), model.Key.from_referable(mlp2)) with self.assertRaises(ValueError) as cm: model.Key.from_referable(mlp1) - self.assertEqual("Can't create Key for MultiLanguageProperty without an id_short!", str(cm.exception)) + self.assertEqual("Can't create Key value for MultiLanguageProperty without an id_short!", str(cm.exception)) mlp1.id_short = "mlp1" self.assertEqual(model.Key(model.KeyTypes.MULTI_LANGUAGE_PROPERTY, "mlp1"), model.Key.from_referable(mlp1)) @@ -51,7 +51,7 @@ def __init__(self): super().__init__() -class ExampleRefereableWithNamespace(model.Referable, model.UniqueIdShortNamespace): +class ExampleReferableWithNamespace(model.Referable, model.UniqueIdShortNamespace): def __init__(self): super().__init__() @@ -79,7 +79,7 @@ def generate_example_referable_with_namespace(id_short: model.NameType, :param child: Child to be added to the namespace sets of the Referable :return: The generated Referable """ - referable = ExampleRefereableWithNamespace() + referable = ExampleReferableWithNamespace() referable.id_short = id_short if child: namespace_set = model.NamespaceSet(parent=referable, attribute_names=[("id_short", True)], @@ -135,6 +135,96 @@ def __init__(self, value: model.Referable): self.assertEqual('Referable must have an identifiable as root object and only parents that are referable', str(cm.exception)) + def test_get_identifiable_root(self): + ref_with_no_parent = ExampleReferableWithNamespace() + ref_with_no_parent.id_short = "NotNone" + + identifiable = ExampleIdentifiable() + + ref_child = ExampleReferable() + ref_child.id_short = "Child" + ref_child.parent = identifiable + + list1 = model.SubmodelElementList("List1", model.SubmodelElementList) + list2 = model.SubmodelElementList(None, model.Property, value_type_list_element=model.datatypes.Int) + prop1 = model.Property(None, model.datatypes.Int) + + list1.parent = ref_child + list1.add_referable(list2) + list2.add_referable(prop1) + + self.assertIs(ref_with_no_parent.get_identifiable_root(), None) + self.assertIs(identifiable.get_identifiable_root(), identifiable) + self.assertIs(ref_child.get_identifiable_root(), identifiable) + self.assertIs(list1.get_identifiable_root(), identifiable) + self.assertIs(list2.get_identifiable_root(), identifiable) + self.assertIs(prop1.get_identifiable_root(), identifiable) + + def test_get_id_short_path(self): + """ + Tests the get_id_short_path() method of Referable objects. + + Example structure: + - SMC: MySubmodelElementCollection + - Property: MySubProperty1 + - Property: MySubProperty2 + - SMC: MySubSubmodelElementCollection + - Property: MySubSubProperty1 + - Property: MySubSubProperty2 + - SML: MySubSubmodelElementList1 + - Property: "MySubTestValue1" + - Property: "MySubTestValue2" + - SML: MySubSubmodelElementList2 + - SML: MySubSubmodelElementList3 + - SMC: MySubmodelElementCollectionInSML3 + - Property: "MySubTestValue3" + """ + MySubmodelElementCollection = model.SubmodelElementCollection("MySubmodelElementCollection") + MySubProperty1 = model.Property("MySubProperty1", model.datatypes.String) + MySubProperty2 = model.Property("MySubProperty2", model.datatypes.String) + MySubSubmodelElementCollection = model.SubmodelElementCollection("MySubSubmodelElementCollection") + MySubSubProperty1 = model.Property("MySubSubProperty1", model.datatypes.String) + MySubSubProperty2 = model.Property("MySubSubProperty2", model.datatypes.String) + MySubSubmodelElementList1 = model.SubmodelElementList("MySubSubmodelElementList1", model.Property, + value_type_list_element=model.datatypes.String) + MySubTestValue1 = model.Property(None, model.datatypes.String) + MySubTestValue2 = model.Property(None, model.datatypes.String) + MySubSubmodelElementList2 = model.SubmodelElementList("MySubSubmodelElementList2", model.SubmodelElementList) + MySubSubmodelElementList3 = model.SubmodelElementList(None, model.SubmodelElementCollection) + MySubmodelElementCollectionInSML3 = model.SubmodelElementCollection(None) + MySubTestValue3 = model.Property("MySubTestValue3", model.datatypes.String) + + MySubmodelElementCollection.add_referable(MySubProperty1) + MySubmodelElementCollection.add_referable(MySubProperty2) + MySubmodelElementCollection.add_referable(MySubSubmodelElementCollection) + MySubSubmodelElementCollection.add_referable(MySubSubProperty1) + MySubSubmodelElementCollection.add_referable(MySubSubProperty2) + MySubmodelElementCollection.add_referable(MySubSubmodelElementList1) + MySubSubmodelElementList1.add_referable(MySubTestValue1) + MySubSubmodelElementList1.add_referable(MySubTestValue2) + MySubmodelElementCollection.add_referable(MySubSubmodelElementList2) + MySubSubmodelElementList2.add_referable(MySubSubmodelElementList3) + MySubSubmodelElementList3.add_referable(MySubmodelElementCollectionInSML3) + MySubmodelElementCollectionInSML3.add_referable(MySubTestValue3) + + expected_id_short_paths = { + MySubmodelElementCollection: "MySubmodelElementCollection", + MySubProperty1: "MySubmodelElementCollection.MySubProperty1", + MySubProperty2: "MySubmodelElementCollection.MySubProperty2", + MySubSubmodelElementCollection: "MySubmodelElementCollection.MySubSubmodelElementCollection", + MySubSubProperty1: "MySubmodelElementCollection.MySubSubmodelElementCollection.MySubSubProperty1", + MySubSubProperty2: "MySubmodelElementCollection.MySubSubmodelElementCollection.MySubSubProperty2", + MySubSubmodelElementList1: "MySubmodelElementCollection.MySubSubmodelElementList1", + MySubTestValue1: "MySubmodelElementCollection.MySubSubmodelElementList1[0]", + MySubTestValue2: "MySubmodelElementCollection.MySubSubmodelElementList1[1]", + MySubSubmodelElementList2: "MySubmodelElementCollection.MySubSubmodelElementList2", + MySubSubmodelElementList3: "MySubmodelElementCollection.MySubSubmodelElementList2[0]", + MySubmodelElementCollectionInSML3: "MySubmodelElementCollection.MySubSubmodelElementList2[0][0]", + MySubTestValue3: "MySubmodelElementCollection.MySubSubmodelElementList2[0][0].MySubTestValue3", + } + for referable, expected_path in expected_id_short_paths.items(): + self.assertEqual(referable.get_id_short_path(), expected_path) + def test_update_from(self): example_submodel = example_aas.create_example_submodel() example_relel = example_submodel.get_referable('ExampleRelationshipElement') @@ -481,7 +571,7 @@ def test_id_short_path_resolution(self) -> None: with self.assertRaises(TypeError) as cm_3: self.namespace.get_referable(["List1", "0", "Prop1", "Test"]) self.assertEqual("Cannot resolve id_short or index 'Test' at " - f"Property[{self.namespace.id} / List1[0] / Prop1], " + f"Property[{self.namespace.id} / List1[0].Prop1], " "because it is not a UniqueIdShortNamespace!", str(cm_3.exception)) self.namespace.get_referable(["List1", "0", "Prop1"]) @@ -568,7 +658,7 @@ def test_aasd_117(self) -> None: se_collection.add_referable(property) with self.assertRaises(model.AASConstraintViolation) as cm: property.id_short = None - self.assertEqual("id_short of Property[foo / property] cannot be unset, since it is already contained in " + self.assertEqual("id_short of Property[foo.property] cannot be unset, since it is already contained in " "SubmodelElementCollection[foo] (Constraint AASd-117)", str(cm.exception)) property.id_short = "bar" @@ -819,7 +909,7 @@ def get_identifiable(self, identifier: Identifier) -> Identifiable: model.Property) with self.assertRaises(TypeError) as cm_3: ref4.resolve(DummyObjectProvider()) - self.assertEqual("Cannot resolve id_short or index 'prop' at Property[urn:x-test:submodel / list[0] / prop], " + self.assertEqual("Cannot resolve id_short or index 'prop' at Property[urn:x-test:submodel / list[0].prop], " "because it is not a UniqueIdShortNamespace!", str(cm_3.exception)) with self.assertRaises(AttributeError) as cm_4: @@ -896,13 +986,13 @@ def test_from_referable(self) -> None: submodel.submodel_element.remove(collection) with self.assertRaises(ValueError) as cm: ref3 = model.ModelReference.from_referable(prop) - self.assertEqual("The given Referable object is not embedded within an Identifiable object", str(cm.exception)) + self.assertEqual("The given Referable object is not embedded within an Identifiable object", + str(cm.exception).split(":")[0]) - # Test creating a reference to a custom Referable class - class DummyThing(model.Referable): + # Test creating a reference to a custom SubmodelElement class + class DummyThing(model.SubmodelElement): def __init__(self, id_short: model.NameType): - super().__init__() - self.id_short = id_short + super().__init__(id_short) class DummyIdentifyableNamespace(model.Submodel, model.UniqueIdShortNamespace): def __init__(self, id_: model.Identifier): @@ -913,7 +1003,7 @@ def __init__(self, id_: model.Identifier): identifable_thing = DummyIdentifyableNamespace("urn:x-test:thing") identifable_thing.things.add(thing) ref4 = model.ModelReference.from_referable(thing) - self.assertIs(ref4.type, model.Referable) + self.assertIs(ref4.type, model.SubmodelElement) class AdministrativeInformationTest(unittest.TestCase): diff --git a/server/Dockerfile b/server/Dockerfile index 059b2e8ab..7ad70bc66 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -23,7 +23,7 @@ RUN chmod +x /etc/supervisor/stop-supervisor.sh # Makes it possible to use a different configuration ENV UWSGI_INI=/etc/uwsgi/uwsgi.ini -# object stores aren't thread-safe yet +# Object stores aren't thread-safe yet # https://github.com/eclipse-basyx/basyx-python-sdk/issues/205 ENV UWSGI_CHEAPER=0 ENV UWSGI_PROCESSES=1 @@ -31,6 +31,14 @@ ENV NGINX_MAX_UPLOAD=1M ENV NGINX_WORKER_PROCESSES=1 ENV LISTEN_PORT=80 ENV CLIENT_BODY_BUFFER_SIZE=1M +ENV API_BASE_PATH=/api/v3.0/ + +# Default values for the storage envs +ENV INPUT=/input +ENV STORAGE=/storage +ENV STORAGE_PERSISTENCY=False +ENV STORAGE_OVERWRITE=False +VOLUME ["/input", "/storage"] # Copy the entrypoint that will generate Nginx additional configs COPY server/entrypoint.sh /entrypoint.sh diff --git a/server/README.md b/server/README.md index 979771cf1..888024ab3 100644 --- a/server/README.md +++ b/server/README.md @@ -6,10 +6,10 @@ The server currently implements the following interfaces: - [Asset Administration Shell Repository Service][4] - [Submodel Repository Service][5] -It uses the [HTTP API][1] and the [AASX][7], [JSON][8], and [XML][9] Adapters of the [BaSyx Python SDK][3], to serve regarding files from a given directory. +It uses the [HTTP API][1] and the [*AASX*][7], [*JSON*][8], and [*XML*][9] Adapters of the [BaSyx Python SDK][3], to serve regarding files from a given directory. The files are only read, changes won't persist. -Alternatively, the container can also be told to use the [Local-File Backend][2] instead, which stores AAS and Submodels as individual JSON files and allows for persistent changes (except supplementary files, i.e. files referenced by `File` submodel elements). +Alternatively, the container can also be told to use the [Local-File Backend][2] instead, which stores Asset Administration Shells (AAS) and Submodels as individual *JSON* files and allows for persistent changes (except supplementary files, i.e. files referenced by `File` SubmodelElements). See [below](#options) on how to configure this. ## Building @@ -19,17 +19,20 @@ The container image can be built via: $ docker build -t basyx-python-server -f Dockerfile .. ``` -Note that when cloning this repository on Windows, Git may convert the line separators to CRLF. This breaks `entrypoint.sh` and `stop-supervisor.sh`. Ensure both files use LF line separators before building. +Note that when cloning this repository on Windows, Git may convert the line separators to CRLF. This breaks [`entrypoint.sh`](entrypoint.sh) and [`stop-supervisor.sh`](stop-supervisor.sh). Ensure both files use LF line separators (`\n`) before building. ## Running ### Storage -The container needs to be provided with the directory `/storage` to store AAS and Submodel files: AASX, JSON, XML or JSON files of Local-File Backend. +The server makes use of two directories: -This directory can be mapped via the `-v` option from another image or a local directory. -To map the directory `storage` inside the container, `-v ./storage:/storage` can be used. -The directory `storage` will be created in the current working directory, if it doesn't already exist. +- **`/input`** - *start-up data*: Directory from which the server loads AAS and Submodel files in *AASX*, *JSON* or *XML* format during start-up. The server will not modify these files. +- **`/storage`** - *persistent store*: Directory where all AAS and Submodels are stored as individual *JSON* files if the server is [configured](#options) for persistence. The server will modify these files. + +The directories can be mapped via the `-v` option from another image or a local directory. +To mount the host directories into the container, `-v ./input:/input -v ./storage:/storage` can be used. +Both local directories `./input` and `./storage` will be created in the current working directory, if they don't already exist. ### Port @@ -38,31 +41,40 @@ To expose it on the host on port 8080, use the option `-p 8080:80` when running ### Options -The container can be configured via environment variables: -- `API_BASE_PATH` determines the base path under which all other API paths are made available. - Default: `/api/v3.0` -- `STORAGE_TYPE` can be one of `LOCAL_FILE_READ_ONLY` or `LOCAL_FILE_BACKEND`: - - When set to `LOCAL_FILE_READ_ONLY` (the default), the server will read and serve AASX, JSON, XML files from the storage directory. - The files are not modified, all changes done via the API are only stored in memory. - - When instead set to `LOCAL_FILE`, the server makes use of the [LocalFileBackend][2], where AAS and Submodels are persistently stored as JSON files. - Supplementary files, i.e. files referenced by `File` submodel elements, are not stored in this case. -- `STORAGE_PATH` sets the directory to read the files from *within the container*. If you bind your files to a directory different from the default `/storage`, you can use this variable to adjust the server accordingly. +The container can be configured via environment variables. The most important ones are summarised below: + +| Variable | Description | Default | +|-----------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------| +| `API_BASE_PATH` | Base path under which the API is served. | `/api/v3.0/` | +| `INPUT` | Path inside the container pointing to the directory from which the server takes its start-up data (*AASX*, *JSON*, *XML*). | `/input` | +| `STORAGE` | Path inside the container pointing to the directory used by the server to persistently store data (*JSON*). | `/storage` | +| `STORAGE_PERSISTENCY` | Flag to enable data persistence via the [LocalFileBackend][2]. AAS/Submodels are stored as *JSON* files in the directory specified by `STORAGE`. Supplementary files, i.e. files referenced by `File` SubmodelElements, are not stored. If disabled, any changes made via the API are only stored in memory. | `False` | +| `STORAGE_OVERWRITE` | Flag to enable storage overwrite if `STORAGE_PERSISTENCY` is enabled. Any AAS/Submodel from the `INPUT` directory already present in the LocalFileBackend replaces its existing version. If disabled, the existing version is kept. | `False` | + + +This implies the following start-up behaviour: + +- Any AAS/Submodel found in `INPUT` is loaded during start-up. +- If `STORAGE_PERSISTENCY = True`: + - Any AAS/Submodel *not* present in the LocalFileBackend is added to it. + - Any AAS/Submodel *already present* is skipped, unless `STORAGE_OVERWRITE = True`, in which case it is replaced. +- Supplementary files (e.g., `File` SubmodelElements) are never persisted by the LocalFileBackend. ### Running Examples Putting it all together, the container can be started via the following command: ``` -$ docker run -p 8080:80 -v ./storage:/storage basyx-python-server +$ docker run -p 8080:80 -v ./input:/input -v ./storage:/storage basyx-python-server ``` Since Windows uses backslashes instead of forward slashes in paths, you'll have to adjust the path to the storage directory there: ``` -> docker run -p 8080:80 -v .\storage:/storage basyx-python-server +> docker run -p 8080:80 -v .\input:/input -v .\storage:/storage basyx-python-server ``` -Per default, the server will use the `LOCAL_FILE_READ_ONLY` storage type and serve the API under `/api/v3.0` and read files from `/storage`. If you want to change this, you can do so like this: +By default, the server will use the standard settings described [above](#options). Those settings can be adapted in the following way: ``` -$ docker run -p 8080:80 -v ./storage2:/storage2 -e API_BASE_PATH=/api/v3.1 -e STORAGE_TYPE=LOCAL_FILE_BACKEND -e STORAGE_PATH=/storage2 basyx-python-server +$ docker run -p 8080:80 -v ./input:/input2 -v ./storage:/storage2 -e API_BASE_PATH=/api/v3.1/ -e INPUT=/input2 -e STORAGE=/storage2 -e STORAGE_PERSISTENCY=True -e STORAGE_OVERWRITE=True basyx-python-server ``` ## Building and Running the Image with Docker Compose @@ -72,8 +84,9 @@ The container image can also be built and run via: $ docker compose up ``` -This is the exemplary `compose.yml` file for the server: +An exemplary [`compose.yml`](compose.yml) file for the server is given [here](compose.yml): ```yaml +name: basyx-python-server services: app: build: @@ -82,13 +95,16 @@ services: ports: - "8080:80" volumes: + - ./input:/input - ./storage:/storage + environment: + STORAGE_PERSISTENCY: True ``` -Here files are read from `/storage` and the server can be accessed at http://localhost:8080/api/v3.0/ from your host system. -To get a different setup this compose.yaml file can be adapted and expanded. +Input files are read from `./input` and stored persistently under `./storage` on your host system. The server can be accessed at http://localhost:8080/api/v3.0/ from your host system. +To get a different setup, the [`compose.yml`](compose.yml) file can be adapted using the options described [above](#options), similar to the third [running example](#running-examples). -Note that the `Dockerfile` has to be specified explicitly, as the build context must be set to the parent directory of `/server` to allow access to the local `/sdk`. +Note that the `Dockerfile` has to be specified explicitly via `dockerfile: server/Dockerfile`, as the build context must be set to the parent directory of `/server` to allow access to the local `/sdk`. ## Running without Docker (Debugging Only) @@ -103,7 +119,7 @@ The server can also be run directly on the host system without Docker, NGINX and $ pip install ./app ``` -2. Run the server by executing the main function in [`./app/interfaces/repository.py`](./app/interfaces/repository.py) from within the current folder. +2. Run the server by executing the main function in [`./app/interfaces/repository.py`](./app/interfaces/repository.py). ```bash $ python -m app.interfaces.repository ``` @@ -119,7 +135,7 @@ This Dockerfile is inspired by the [tiangolo/uwsgi-nginx-docker][10] repository. [3]: https://github.com/eclipse-basyx/basyx-python-sdk [4]: https://app.swaggerhub.com/apis/Plattform_i40/AssetAdministrationShellRepositoryServiceSpecification/V3.0.1_SSP-001 [5]: https://app.swaggerhub.com/apis/Plattform_i40/SubmodelRepositoryServiceSpecification/V3.0.1_SSP-001 -[6]: https://industrialdigitaltwin.org/content-hub/aasspecifications/idta_01002-3-0_application_programming_interfaces +[6]: https://industrialdigitaltwin.io/aas-specifications/IDTA-01002/v3.0/index.html [7]: https://basyx-python-sdk.readthedocs.io/en/latest/adapter/aasx.html#adapter-aasx [8]: https://basyx-python-sdk.readthedocs.io/en/latest/adapter/json.html [9]: https://basyx-python-sdk.readthedocs.io/en/latest/adapter/xml.html diff --git a/server/app/interfaces/base.py b/server/app/interfaces/base.py index 686bb92cb..2f23b5622 100644 --- a/server/app/interfaces/base.py +++ b/server/app/interfaces/base.py @@ -287,6 +287,7 @@ class HTTPApiDecoder: model.Submodel: XMLConstructables.SUBMODEL, model.SubmodelElement: XMLConstructables.SUBMODEL_ELEMENT, model.Reference: XMLConstructables.REFERENCE, + model.ConceptDescription: XMLConstructables.CONCEPT_DESCRIPTION, } @classmethod diff --git a/server/app/main.py b/server/app/main.py index 56ae4f59c..49920f628 100644 --- a/server/app/main.py +++ b/server/app/main.py @@ -1,48 +1,126 @@ -import os -import pathlib -import sys - -from basyx.aas import model -from basyx.aas.adapter.aasx import AASXReader, DictSupplementaryFileContainer -from basyx.aas.adapter.json import read_aas_json_file_into -from basyx.aas.adapter.xml import read_aas_xml_file_into +# Copyright (c) 2025 the Eclipse BaSyx Authors +# +# This program and the accompanying materials are made available under the terms of the MIT License, available in +# the LICENSE file of this project. +# +# SPDX-License-Identifier: MIT +""" +This module provides the WSGI entry point for the Asset Administration Shell Repository Server. +""" +import logging +import os +from basyx.aas.adapter import load_directory +from basyx.aas.adapter.aasx import DictSupplementaryFileContainer from basyx.aas.backend.local_file import LocalFileObjectStore +from basyx.aas.model.provider import DictObjectStore from interfaces.repository import WSGIApp +from typing import Tuple, Union + + +# -------- Helper methods -------- + +def setup_logger() -> logging.Logger: + """ + Configure a custom :class:`~logging.Logger` for the start-up sequence of the server. + + :return: Configured :class:`~logging.Logger` + """ + + logger = logging.getLogger(__name__) + if not logger.handlers: + logger.setLevel(logging.INFO) + handler = logging.StreamHandler() + handler.setLevel(logging.INFO) + handler.setFormatter(logging.Formatter("%(levelname)s [Server Start-up] %(message)s")) + logger.addHandler(handler) + logger.propagate = False + return logger + + +def build_storage( + env_input: str, + env_storage: str, + env_storage_persistency: bool, + env_storage_overwrite: bool, + logger: logging.Logger +) -> Tuple[Union[DictObjectStore, LocalFileObjectStore], DictSupplementaryFileContainer]: + """ + Configure the server's storage according to the given start-up settings. + + :param env_input: ``str`` pointing to the input directory of the server + :param env_storage: ``str`` pointing to the :class:`~basyx.aas.backend.local_file.LocalFileObjectStore` storage + directory of the server if persistent storage is enabled + :param env_storage_persistency: Flag to enable persistent storage + :param env_storage_overwrite: Flag to overwrite existing :class:`Identifiables ` + in the :class:`~basyx.aas.backend.local_file.LocalFileObjectStore` if persistent storage is enabled + :param logger: :class:`~logging.Logger` used for start-up diagnostics + :return: Tuple consisting of a :class:`~basyx.aas.model.provider.DictObjectStore` if persistent storage is disabled + or a :class:`~basyx.aas.backend.local_file.LocalFileObjectStore` if persistent storage is enabled and a + :class:`~basyx.aas.adapter.aasx.DictSupplementaryFileContainer` as storage for + :class:`~interfaces.repository.WSGIApp` + """ + + if env_storage_persistency: + storage_files = LocalFileObjectStore(env_storage) + storage_files.check_directory(create=True) + if os.path.isdir(env_input): + input_files, input_supp_files = load_directory(env_input) + added, overwritten, skipped = storage_files.sync(input_files, env_storage_overwrite) + logger.info( + "Loaded %d identifiable(s) and %d supplementary file(s) from \"%s\"", + len(input_files), len(input_supp_files), env_input + ) + logger.info( + "Synced INPUT to STORAGE with %d added and %d %s", + added, + overwritten if env_storage_overwrite else skipped, + "overwritten" if env_storage_overwrite else "skipped" + ) + return storage_files, input_supp_files + else: + logger.warning("INPUT directory \"%s\" not found, starting empty", env_input) + return storage_files, DictSupplementaryFileContainer() + + if os.path.isdir(env_input): + input_files, input_supp_files = load_directory(env_input) + logger.info( + "Loaded %d identifiable(s) and %d supplementary file(s) from \"%s\"", + len(input_files), len(input_supp_files), env_input + ) + return input_files, input_supp_files + else: + logger.warning("INPUT directory \"%s\" not found, starting empty", env_input) + return DictObjectStore(), DictSupplementaryFileContainer() + -storage_path = os.getenv("STORAGE_PATH", "/storage") -storage_type = os.getenv("STORAGE_TYPE", "LOCAL_FILE_READ_ONLY") -base_path = os.getenv("API_BASE_PATH") +# -------- WSGI entrypoint -------- -wsgi_optparams = {} +logger = setup_logger() -if base_path is not None: - wsgi_optparams["base_path"] = base_path +env_input = os.getenv("INPUT", "/input") +env_storage = os.getenv("STORAGE", "/storage") +env_storage_persistency = os.getenv("STORAGE_PERSISTENCY", "false").lower() in {"1", "true", "yes"} +env_storage_overwrite = os.getenv("STORAGE_OVERWRITE", "false").lower() in {"1", "true", "yes"} +env_api_base_path = os.getenv("API_BASE_PATH") -if storage_type == "LOCAL_FILE_BACKEND": - application = WSGIApp(LocalFileObjectStore(storage_path), DictSupplementaryFileContainer(), **wsgi_optparams) +wsgi_optparams = {"base_path": env_api_base_path} if env_api_base_path else {} -elif storage_type == "LOCAL_FILE_READ_ONLY": - object_store: model.DictObjectStore = model.DictObjectStore() - file_store: DictSupplementaryFileContainer = DictSupplementaryFileContainer() +logger.info( + "Loaded settings API_BASE_PATH=\"%s\", INPUT=\"%s\", STORAGE=\"%s\", PERSISTENCY=%s, OVERWRITE=%s", + env_api_base_path or "", env_input, env_storage, env_storage_persistency, env_storage_overwrite +) - for file in pathlib.Path(storage_path).iterdir(): - if not file.is_file(): - continue - print(f"Loading {file}") +storage_files, supp_files = build_storage( + env_input, + env_storage, + env_storage_persistency, + env_storage_overwrite, + logger +) - if file.suffix.lower() == ".json": - with open(file) as f: - read_aas_json_file_into(object_store, f) - elif file.suffix.lower() == ".xml": - with open(file) as f: - read_aas_xml_file_into(object_store, f) - elif file.suffix.lower() == ".aasx": - with AASXReader(file) as reader: - reader.read_into(object_store=object_store, file_store=file_store) +application = WSGIApp(storage_files, supp_files, **wsgi_optparams) - application = WSGIApp(object_store, file_store, **wsgi_optparams) -else: - print(f"STORAGE_TYPE must be either LOCAL_FILE or LOCAL_FILE_READ_ONLY! Current value: {storage_type}", - file=sys.stderr) +if __name__ == "__main__": + logger.info("WSGI entrypoint created. Serve this module with uWSGI/Gunicorn/etc.") diff --git a/server/app/util/converters.py b/server/app/util/converters.py index 0db897cb6..4e37c4702 100644 --- a/server/app/util/converters.py +++ b/server/app/util/converters.py @@ -60,24 +60,20 @@ def to_python(self, value: str) -> model.Identifier: class IdShortPathConverter(werkzeug.routing.UnicodeConverter): """ - A custom Werkzeug URL converter for handling id_short_sep-separated idShort paths. + A custom Werkzeug URL converter for handling dot-separated idShort paths and indexes. This converter joins a list of idShort strings into an id_short_sep-separated path for URLs - (e.g., ["submodel", "element"] -> "submodel.element") and parses incoming URL paths + (e.g., ["submodel", "element", "1"] -> "submodel.element[1]") and parses incoming URL paths back into a list, validating each idShort. - - :cvar id_short_sep: Separator used to join and split idShort segments. """ - id_short_sep = "." def to_url(self, value: List[str]) -> str: - return super().to_url(self.id_short_sep.join(value)) + id_short_path = model.Referable.build_id_short_path(value) + return super().to_url(id_short_path) def to_python(self, value: str) -> List[str]: - id_shorts = super().to_python(value).split(self.id_short_sep) - for id_short in id_shorts: - try: - model.Referable.validate_id_short(id_short) - except (ValueError, model.AASConstraintViolation): - raise BadRequest(f"{id_short} is not a valid id_short!") - return id_shorts + try: + parsed_id_short_path = model.Referable.parse_id_short_path(value) + except (ValueError, model.AASConstraintViolation) as e: + raise BadRequest(f"{value} is not a valid id_short!") from e + return parsed_id_short_path diff --git a/server/compose.yml b/server/compose.yml index 666484a5d..f7e014c37 100644 --- a/server/compose.yml +++ b/server/compose.yml @@ -1,3 +1,4 @@ +name: basyx-python-server services: app: build: @@ -6,4 +7,7 @@ services: ports: - "8080:80" volumes: + - ./input:/input - ./storage:/storage + environment: + STORAGE_PERSISTENCY: True