diff --git a/.github/workflows/test-and-release.yml b/.github/workflows/test-and-release.yml index 705c4db..6df89b9 100644 --- a/.github/workflows/test-and-release.yml +++ b/.github/workflows/test-and-release.yml @@ -1,11 +1,6 @@ name: 'Test and Release PyDSDL' on: [ push, pull_request ] -# Ensures that only one workflow is running at a time -concurrency: - group: ${{ github.workflow_sha }} - cancel-in-progress: true - jobs: pydsdl-test: name: Test PyDSDL diff --git a/noxfile.py b/noxfile.py index 52b9780..376798d 100644 --- a/noxfile.py +++ b/noxfile.py @@ -98,7 +98,7 @@ def lint(session): ) if is_latest_python(session): # we run black only on the newest Python version to ensure that the code is formatted with the latest version - session.install("black ~= 24.4") + session.install("black ~= 25.1") session.run("black", "--check", ".") diff --git a/pydsdl/__init__.py b/pydsdl/__init__.py index 5f9ac61..d0292a2 100644 --- a/pydsdl/__init__.py +++ b/pydsdl/__init__.py @@ -7,11 +7,11 @@ import sys as _sys from pathlib import Path as _Path -__version__ = "1.22.2" +__version__ = "1.23.0" __version_info__ = tuple(map(int, __version__.split(".")[:3])) __license__ = "MIT" __author__ = "OpenCyphal" -__copyright__ = "Copyright (c) 2018 OpenCyphal" +__copyright__ = "Copyright (c) OpenCyphal development team" __email__ = "maintainers@opencyphal.org" # Our unorthodox approach to dependency management requires us to apply certain workarounds. @@ -42,6 +42,8 @@ from ._serializable import IntegerType as IntegerType from ._serializable import SignedIntegerType as SignedIntegerType from ._serializable import UnsignedIntegerType as UnsignedIntegerType +from ._serializable import ByteType as ByteType +from ._serializable import UTF8Type as UTF8Type from ._serializable import FloatType as FloatType from ._serializable import VoidType as VoidType from ._serializable import ArrayType as ArrayType diff --git a/pydsdl/_bit_length_set/_bit_length_set.py b/pydsdl/_bit_length_set/_bit_length_set.py index 59cc024..fb4299f 100644 --- a/pydsdl/_bit_length_set/_bit_length_set.py +++ b/pydsdl/_bit_length_set/_bit_length_set.py @@ -347,7 +347,7 @@ def elementwise_sum_k_multicombinations(self, k: int) -> "BitLengthSet": # prag @staticmethod def elementwise_sum_cartesian_product( - sets: typing.Iterable[typing.Union[typing.Iterable[int], int]] + sets: typing.Iterable[typing.Union[typing.Iterable[int], int]], ) -> "BitLengthSet": # pragma: no cover """ :meta private: diff --git a/pydsdl/_dsdl.py b/pydsdl/_dsdl.py index f8d7c98..96153a8 100644 --- a/pydsdl/_dsdl.py +++ b/pydsdl/_dsdl.py @@ -119,6 +119,8 @@ def read( definition_visitors: Iterable["DefinitionVisitor"], print_output_handler: Callable[[int, str], None], allow_unregulated_fixed_port_id: bool, + *, + strict: bool = False, ) -> CompositeType: """ Reads the data type definition and returns its high-level data type representation. @@ -133,6 +135,7 @@ def read( :param definition_visitors: Visitors to notify about discovered dependencies. :param print_output_handler: Used for @print and for diagnostics: (line_number, text) -> None. :param allow_unregulated_fixed_port_id: Do not complain about fixed unregulated port IDs. + :param strict: Reject features that are not part of the Specification. :return: The data type representation. """ raise NotImplementedError() diff --git a/pydsdl/_dsdl_definition.py b/pydsdl/_dsdl_definition.py index 097cf7c..3caada3 100644 --- a/pydsdl/_dsdl_definition.py +++ b/pydsdl/_dsdl_definition.py @@ -235,6 +235,8 @@ def read( definition_visitors: Iterable[DefinitionVisitor], print_output_handler: Callable[[int, str], None], allow_unregulated_fixed_port_id: bool, + *, + strict: bool = False, ) -> CompositeType: log_prefix = "%s.%d.%d" % (self.full_name, self.version.major, self.version.minor) if self._cached_type is not None: @@ -262,7 +264,7 @@ def read( allow_unregulated_fixed_port_id=allow_unregulated_fixed_port_id, ) - _parser.parse(self.text, builder) + _parser.parse(self.text, builder, strict=strict) self._cached_type = builder.finalize() _logger.info( @@ -536,5 +538,5 @@ def _unittest_type_from_path_inference_edge_case(temp_dsdl_factory) -> None: # def _unittest_from_first_in(temp_dsdl_factory) -> None: # type: ignore dsdl_file = temp_dsdl_factory.new_file(Path("repo/uavcan/foo/bar/435.baz.1.0.dsdl"), "@sealed") - dsdl_def = DSDLDefinition.from_first_in(dsdl_file.resolve(), [(dsdl_file.parent.parent / "..")]) + dsdl_def = DSDLDefinition.from_first_in(dsdl_file.resolve(), [dsdl_file.parent.parent / ".."]) assert dsdl_def.full_name == "uavcan.foo.bar.baz" diff --git a/pydsdl/_expression/_container.py b/pydsdl/_expression/_container.py index 7135705..2d05de7 100644 --- a/pydsdl/_expression/_container.py +++ b/pydsdl/_expression/_container.py @@ -32,7 +32,7 @@ class Set(Container): class _Decorator: @staticmethod def homotypic_binary_operator( - inferior: typing.Callable[["Set", "Set"], _O] + inferior: typing.Callable[["Set", "Set"], _O], ) -> typing.Callable[["Set", "Set"], _O]: def wrapper(self: "Set", other: "Set") -> _O: assert isinstance(self, Set) and isinstance(other, Set) diff --git a/pydsdl/_namespace.py b/pydsdl/_namespace.py index 02d8306..50259b8 100644 --- a/pydsdl/_namespace.py +++ b/pydsdl/_namespace.py @@ -74,6 +74,8 @@ def read_namespace( print_output_handler: PrintOutputHandler | None = None, allow_unregulated_fixed_port_id: bool = False, allow_root_namespace_name_collision: bool = True, + *, + strict: bool = False, ) -> list[_serializable.CompositeType]: """ This function is a main entry point for the library. @@ -103,6 +105,8 @@ def read_namespace( the same root namespace name multiple times in the lookup dirs. This will enable defining a namespace partially and let other entities define new messages or new sub-namespaces in the same root namespace. + :param strict: Reject features that are not [yet] part of the Cyphal Specification. + :return: A list of :class:`pydsdl.CompositeType` found under the `root_namespace_directory` and sorted lexicographically by full data type name, then by major version (newest version first), then by minor version (newest version first). The ordering guarantee allows the caller to always find the newest version @@ -131,7 +135,11 @@ def read_namespace( _logger.debug(_LOG_LIST_ITEM_PREFIX + str(x)) return _complete_read_function( - target_dsdl_definitions, lookup_directories_path_list, print_output_handler, allow_unregulated_fixed_port_id + target_dsdl_definitions, + lookup_directories_path_list, + print_output_handler, + allow_unregulated_fixed_port_id=allow_unregulated_fixed_port_id, + strict=strict, ).direct @@ -142,6 +150,8 @@ def read_files( lookup_directories: None | Path | str | Iterable[Path | str] = None, print_output_handler: PrintOutputHandler | None = None, allow_unregulated_fixed_port_id: bool = False, + *, + strict: bool = False, ) -> tuple[list[_serializable.CompositeType], list[_serializable.CompositeType]]: """ This function is a main entry point for the library. @@ -232,6 +242,8 @@ def read_files( This is a dangerous feature that must not be used unless you understand the risks. Please read https://opencyphal.org/guide. + :param strict: Reject features that are not [yet] part of the Cyphal Specification. + :return: A Tuple of lists of :class:`pydsdl.CompositeType`. The first index in the Tuple are the types parsed from the ``dsdl_files`` argument. The second index are types that the target ``dsdl_files`` utilizes. A note for using these values to describe build dependencies: each :class:`pydsdl.CompositeType` has two @@ -266,9 +278,12 @@ def read_files( ) definitions = _complete_read_function( - target_dsdl_definitions, lookup_directories_path_list, print_output_handler, allow_unregulated_fixed_port_id + target_dsdl_definitions, + lookup_directories_path_list, + print_output_handler, + allow_unregulated_fixed_port_id=allow_unregulated_fixed_port_id, + strict=strict, ) - return (definitions.direct, definitions.transitive) @@ -286,7 +301,9 @@ def _complete_read_function( target_dsdl_definitions: SortedFileList[ReadableDSDLFile], lookup_directories_path_list: list[Path], print_output_handler: PrintOutputHandler | None, + *, allow_unregulated_fixed_port_id: bool, + strict: bool, ) -> DSDLDefinitions: lookup_dsdl_definitions = _construct_dsdl_definitions_from_namespaces(lookup_directories_path_list) @@ -307,7 +324,11 @@ def _complete_read_function( # This is the biggie. All the rest of the wrangling is just to get to this point. This will take the # most time and memory. definitions = read_definitions( - target_dsdl_definitions, lookup_dsdl_definitions, print_output_handler, allow_unregulated_fixed_port_id + target_dsdl_definitions, + lookup_dsdl_definitions, + print_output_handler, + allow_unregulated_fixed_port_id=allow_unregulated_fixed_port_id, + strict=strict, ) # Note that we check for collisions in the read namespace only. diff --git a/pydsdl/_namespace_reader.py b/pydsdl/_namespace_reader.py index fd264e1..7222431 100644 --- a/pydsdl/_namespace_reader.py +++ b/pydsdl/_namespace_reader.py @@ -19,12 +19,14 @@ def _read_definitions( target_definitions: SortedFileList[ReadableDSDLFile], lookup_definitions: SortedFileList[ReadableDSDLFile], + *, print_output_handler: PrintOutputHandler | None, allow_unregulated_fixed_port_id: bool, + strict: bool, direct: set[CompositeType], transitive: set[CompositeType], file_pool: dict[Path, ReadableDSDLFile], - level: int, + level: int = 0, ) -> None: """ Don't look at me! I'm hideous! @@ -87,12 +89,13 @@ def print_handler(file: Path, line: int, message: str) -> None: _read_definitions( dsdl_file_sort(_pending_definitions), lookup_definitions, - print_output_handler, - allow_unregulated_fixed_port_id, - direct, - transitive, - file_pool, - level + 1, + print_output_handler=print_output_handler, + allow_unregulated_fixed_port_id=allow_unregulated_fixed_port_id, + strict=strict, + direct=direct, + transitive=transitive, + file_pool=file_pool, + level=level + 1, ) _pending_definitions.clear() @@ -116,6 +119,8 @@ def read_definitions( lookup_definitions: SortedFileList[ReadableDSDLFile], print_output_handler: PrintOutputHandler | None, allow_unregulated_fixed_port_id: bool, + *, + strict: bool = False, ) -> DSDLDefinitions: """ Given a set of DSDL files, this method reads the text and invokes the parser for each and for any files found in the @@ -125,6 +130,7 @@ def read_definitions( :param lookup_definitions: List of definitions available for referring to. :param print_output_handler: Used for @print and for diagnostics: (line_number, text) -> None. :param allow_unregulated_fixed_port_id: Do not complain about fixed unregulated port IDs. + :param strict: Reject features that are not part of the Specification. :return: The data type representation. :raises InvalidDefinitionError: If a dependency is missing. :raises InternalError: If an unexpected error occurs. @@ -135,12 +141,12 @@ def read_definitions( _read_definitions( target_definitions, lookup_definitions, - print_output_handler, - allow_unregulated_fixed_port_id, - _direct, - _transitive, - _file_pool, - 0, + print_output_handler=print_output_handler, + allow_unregulated_fixed_port_id=allow_unregulated_fixed_port_id, + strict=strict, + direct=_direct, + transitive=_transitive, + file_pool=_file_pool, ) return DSDLDefinitions( dsdl_file_sort(_direct), @@ -186,6 +192,7 @@ def _unittest_namespace_reader_read_definitions_multiple(temp_dsdl_factory) -> N assert len(definitions.transitive) == 2 +# noinspection PyProtectedMember def _unittest_namespace_reader_read_definitions_multiple_no_load(temp_dsdl_factory) -> None: # type: ignore """ Ensure that the loader does not load files that are not in the transitive closure of the target files. diff --git a/pydsdl/_parser.py b/pydsdl/_parser.py index cc25631..7c49eaf 100644 --- a/pydsdl/_parser.py +++ b/pydsdl/_parser.py @@ -21,12 +21,12 @@ class DSDLSyntaxError(_error.InvalidDefinitionError): pass -def parse(text: str, statement_stream_processor: "StatementStreamProcessor") -> None: +def parse(text: str, statement_stream_processor: "StatementStreamProcessor", *, strict: bool) -> None: """ The entry point of the parser. As the text is being parsed, the parser invokes appropriate methods in the statement stream processor. """ - pr = _ParseTreeProcessor(statement_stream_processor) + pr = _ParseTreeProcessor(statement_stream_processor, strict=strict) try: pr.visit(_get_grammar().parse(text)) # type: ignore except _error.FrontendError as ex: @@ -134,12 +134,13 @@ class _ParseTreeProcessor(parsimonious.NodeVisitor): # Beware that those might be propagated from recursive parser instances! unwrapped_exceptions = (_error.FrontendError, SystemError, MemoryError, SystemExit) # type: ignore - def __init__(self, statement_stream_processor: StatementStreamProcessor): + def __init__(self, statement_stream_processor: StatementStreamProcessor, *, strict: bool): assert isinstance(statement_stream_processor, StatementStreamProcessor) self._statement_stream_processor = statement_stream_processor # type: StatementStreamProcessor self._current_line_number = 1 # Lines are numbered from one self._comment = "" self._comment_is_header = True + self._strict = bool(strict) super().__init__() @property @@ -265,6 +266,15 @@ def visit_type_version_specifier(self, _n: _Node, children: _Children) -> _seria assert isinstance(major, _expression.Rational) and isinstance(minor, _expression.Rational) return _serializable.Version(major=major.as_native_integer(), minor=minor.as_native_integer()) + def visit_type_primitive_boolean(self, _n: _Node, _c: _Children) -> _serializable.PrimitiveType: + return _serializable.BooleanType() + + def visit_type_primitive_byte(self, _n: _Node, _c: _Children) -> _serializable.PrimitiveType: + return _serializable.ByteType() + + def visit_type_primitive_utf8(self, _n: _Node, _c: _Children) -> _serializable.PrimitiveType: + return _serializable.UTF8Type() + def visit_type_primitive_truncated(self, _n: _Node, children: _Children) -> _serializable.PrimitiveType: _kw, _sp, cons = cast(Tuple[_Node, _Node, _PrimitiveTypeConstructor], children) return cons(_serializable.PrimitiveType.CastMode.TRUNCATED) @@ -273,9 +283,6 @@ def visit_type_primitive_saturated(self, _n: _Node, children: _Children) -> _ser _, cons = cast(Tuple[_Node, _PrimitiveTypeConstructor], children) return cons(_serializable.PrimitiveType.CastMode.SATURATED) - def visit_type_primitive_name_boolean(self, _n: _Node, _c: _Children) -> _PrimitiveTypeConstructor: - return typing.cast(_PrimitiveTypeConstructor, _serializable.BooleanType) - def visit_type_primitive_name_unsigned_integer(self, _n: _Node, children: _Children) -> _PrimitiveTypeConstructor: return lambda cm: _serializable.UnsignedIntegerType(children[-1], cm) diff --git a/pydsdl/_port_id_ranges.py b/pydsdl/_port_id_ranges.py index b0e4116..f23af37 100644 --- a/pydsdl/_port_id_ranges.py +++ b/pydsdl/_port_id_ranges.py @@ -6,7 +6,7 @@ MAX_SERVICE_ID = 511 -_STANDARD_ROOT_NAMESPACE = "uavcan" +_STANDARD_ROOT_NAMESPACES = {"uavcan", "cyphal"} _STANDARD_MESSAGES = 7168, 8191 _STANDARD_SERVICES = 384, 511 @@ -18,13 +18,13 @@ def is_valid_regulated_subject_id(regulated_id: int, root_namespace: str) -> bool: - is_standard = root_namespace.strip() == _STANDARD_ROOT_NAMESPACE + is_standard = root_namespace.strip() in _STANDARD_ROOT_NAMESPACES lo, hi = _STANDARD_MESSAGES if is_standard else _VENDOR_MESSAGES return lo <= int(regulated_id) <= hi def is_valid_regulated_service_id(regulated_id: int, root_namespace: str) -> bool: - is_standard = root_namespace.strip() == _STANDARD_ROOT_NAMESPACE + is_standard = root_namespace.strip() in _STANDARD_ROOT_NAMESPACES lo, hi = _STANDARD_SERVICES if is_standard else _VENDOR_SERVICES return lo <= int(regulated_id) <= hi diff --git a/pydsdl/_serializable/__init__.py b/pydsdl/_serializable/__init__.py index 4589b85..0a3e640 100644 --- a/pydsdl/_serializable/__init__.py +++ b/pydsdl/_serializable/__init__.py @@ -12,6 +12,8 @@ from ._primitive import IntegerType as IntegerType from ._primitive import SignedIntegerType as SignedIntegerType from ._primitive import UnsignedIntegerType as UnsignedIntegerType +from ._primitive import ByteType as ByteType +from ._primitive import UTF8Type as UTF8Type from ._void import VoidType as VoidType diff --git a/pydsdl/_serializable/_array.py b/pydsdl/_serializable/_array.py index 0223d57..dfd93a1 100644 --- a/pydsdl/_serializable/_array.py +++ b/pydsdl/_serializable/_array.py @@ -5,8 +5,9 @@ import abc import math import typing +import warnings from .._bit_length_set import BitLengthSet -from ._serializable import SerializableType, TypeParameterError +from ._serializable import SerializableType, TypeParameterError, AggregationFailure from ._primitive import UnsignedIntegerType, PrimitiveType @@ -22,6 +23,16 @@ def __init__(self, element_type: SerializableType, capacity: int): if self._capacity < 1: raise InvalidNumberOfElementsError("Array capacity cannot be less than 1") + @property + def deprecated(self) -> bool: + return self.element_type.deprecated + + def _check_aggregation(self, aggregate: "SerializableType") -> typing.Optional[AggregationFailure]: + af = self.element_type._check_aggregation(self) # pylint: disable=protected-access + if af is not None: + return AggregationFailure(self, aggregate, "Element type of %r is not valid: %s" % (str(self), af.message)) + return super()._check_aggregation(aggregate) + @property def element_type(self) -> SerializableType: return self._element_type @@ -36,10 +47,10 @@ def capacity(self) -> int: @property def string_like(self) -> bool: """ - True if the array might contain a text string, in which case it is termed to be "string-like". - A string-like array is a variable-length array of ``uint8``. - See https://github.com/OpenCyphal/specification/issues/51. + **This property is deprecated** and will be removed in a future release. + Replace with an explicit check for ``isinstance(array.element_type, UTF8Type)``. """ + warnings.warn("use isinstance(array.element_type, UTF8Type) instead of string_like", DeprecationWarning) return False @property @@ -149,9 +160,13 @@ def bit_length_set(self) -> BitLengthSet: @property def string_like(self) -> bool: - """See the base class.""" + from ._primitive import UTF8Type + + warnings.warn("use isinstance(array.element_type, UTF8Type) instead of string_like", DeprecationWarning) et = self.element_type # Without this temporary MyPy yields a false positive type error - return isinstance(et, UnsignedIntegerType) and (et.bit_length == self.BITS_PER_BYTE) + return isinstance(et, UTF8Type) or ( + isinstance(et, UnsignedIntegerType) and (et.bit_length == self.BITS_PER_BYTE) + ) @property def length_field_type(self) -> UnsignedIntegerType: diff --git a/pydsdl/_serializable/_attribute.py b/pydsdl/_serializable/_attribute.py index c48e0e3..f194be6 100644 --- a/pydsdl/_serializable/_attribute.py +++ b/pydsdl/_serializable/_attribute.py @@ -155,9 +155,9 @@ def _unittest_attribute() -> None: from pytest import raises from ._primitive import SignedIntegerType - assert str(Field(BooleanType(PrimitiveType.CastMode.SATURATED), "flag")) == "saturated bool flag" + assert str(Field(BooleanType(), "flag")) == "bool flag" assert ( - repr(Field(BooleanType(PrimitiveType.CastMode.SATURATED), "flag")) + repr(Field(BooleanType(), "flag")) == "Field(data_type=BooleanType(bit_length=1, cast_mode=), name='flag')" ) diff --git a/pydsdl/_serializable/_composite.py b/pydsdl/_serializable/_composite.py index 1153cc7..9c29591 100644 --- a/pydsdl/_serializable/_composite.py +++ b/pydsdl/_serializable/_composite.py @@ -6,14 +6,14 @@ import math import typing from pathlib import Path - -from .. import _expression, _port_id_ranges +from .. import _expression +from .. import _port_id_ranges from .._bit_length_set import BitLengthSet -from ._attribute import Attribute, Constant, Field, PaddingField -from ._name import InvalidNameError, check_name +from .._error import InvalidDefinitionError +from ._serializable import SerializableType, TypeParameterError, AggregationFailure +from ._attribute import Attribute, Field, PaddingField, Constant +from ._name import check_name, InvalidNameError from ._primitive import PrimitiveType, UnsignedIntegerType -from ._serializable import SerializableType, TypeParameterError -from ._void import VoidType Version = typing.NamedTuple("Version", [("major", int), ("minor", int)]) @@ -38,7 +38,7 @@ class MalformedUnionError(TypeParameterError): pass -class DeprecatedDependencyError(TypeParameterError): +class AggregationError(InvalidDefinitionError): pass @@ -80,10 +80,8 @@ def __init__( # pylint: disable=too-many-arguments, too-many-locals # Name check if not self._name: raise InvalidNameError("Composite type name cannot be empty") - if self.NAME_COMPONENT_SEPARATOR not in self._name: raise InvalidNameError("Root namespace is not specified") - if len(self._name) > self.MAX_NAME_LENGTH: # TODO # Notice that per the Specification, service request/response types are unnamed, @@ -121,7 +119,6 @@ def search_up_for_root(path: Path, namespace_components: typing.List[str]) -> Pa and (0 <= self._version.minor <= self.MAX_VERSION_NUMBER) and ((self._version.major + self._version.minor) > 0) ) - if not version_valid: raise InvalidVersionError("Invalid version numbers: %s.%s" % (self._version.major, self._version.minor)) @@ -143,17 +140,13 @@ def search_up_for_root(path: Path, namespace_components: typing.List[str]) -> Pa if not (0 <= port_id <= _port_id_ranges.MAX_SUBJECT_ID): raise InvalidFixedPortIDError("Fixed subject ID %r is not valid" % port_id) - # Consistent deprecation check. - # A non-deprecated type cannot be dependent on deprecated types. - # A deprecated type can be dependent on anything. - if not self.deprecated: - for a in self._attributes: - t = a.data_type - if isinstance(t, CompositeType): - if t.deprecated: - raise DeprecatedDependencyError( - "A type cannot depend on deprecated types " "unless it is also deprecated." - ) + # Aggregation check. For example: + # - Types like utf8 and byte cannot be used outside of arrays. + # - A non-deprecated type cannot depend on a deprecated type. + for a in self._attributes: + af = a.data_type._check_aggregation(self) + if af is not None: + raise AggregationError("Type of %r is not a valid field type for %s: %s" % (str(a), self, af.message)) @property def full_name(self) -> str: @@ -216,9 +209,12 @@ def bit_length_set(self) -> BitLengthSet: """ raise NotImplementedError + def _check_aggregation(self, aggregate: "SerializableType") -> typing.Optional[AggregationFailure]: + return super()._check_aggregation(aggregate) + @property def deprecated(self) -> bool: - """Whether the definition is marked ``@deprecated``.""" + """True if the definition is marked ``@deprecated``.""" return self._deprecated @property @@ -411,10 +407,6 @@ def __init__( # pylint: disable=too-many-arguments "A tagged union cannot contain fewer than %d variants" % self.MIN_NUMBER_OF_VARIANTS ) - for a in attributes: - if isinstance(a, PaddingField) or not a.name or isinstance(a.data_type, VoidType): - raise MalformedUnionError("Padding fields not allowed in unions") - self._tag_field_type = UnsignedIntegerType( self._compute_tag_bit_length([x.data_type for x in self.fields]), PrimitiveType.CastMode.TRUNCATED ) @@ -652,6 +644,12 @@ def iterate_fields_with_offsets( base_offset = base_offset + self.delimiter_header_type.bit_length_set return self.inner_type.iterate_fields_with_offsets(base_offset) + def _check_aggregation(self, aggregate: "SerializableType") -> typing.Optional[AggregationFailure]: + af = self.inner_type._check_aggregation(aggregate) # pylint: disable=protected-access + if af is not None: + return af + return super()._check_aggregation(aggregate) + def __repr__(self) -> str: return "%s(inner=%r, extent=%r)" % (self.__class__.__name__, self.inner_type, self.extent) @@ -726,11 +724,10 @@ def iterate_fields_with_offsets( def _unittest_composite_types() -> None: # pylint: disable=too-many-statements from typing import Optional - from pytest import raises - from ._array import FixedLengthArrayType, VariableLengthArrayType from ._primitive import FloatType, SignedIntegerType + from ._void import VoidType def try_name(name: str, file_path: Optional[Path] = None) -> CompositeType: return StructureType( @@ -786,7 +783,7 @@ def try_name(name: str, file_path: Optional[Path] = None) -> CompositeType: has_parent_service=False, ) - with raises(MalformedUnionError, match="(?i).*padding.*"): + with raises(AggregationError, match="(?i).*not a valid field type.*"): UnionType( name="a.A", version=Version(0, 1), @@ -998,6 +995,7 @@ def _unittest_field_iterators() -> None: # pylint: disable=too-many-locals from pytest import raises from ._array import FixedLengthArrayType, VariableLengthArrayType + from ._void import VoidType from ._primitive import BooleanType, FloatType saturated = PrimitiveType.CastMode.SATURATED @@ -1031,7 +1029,7 @@ def validate_iterator( StructureType, [ Field(UnsignedIntegerType(10, saturated), "a"), - Field(BooleanType(saturated), "b"), + Field(BooleanType(), "b"), Field(VariableLengthArrayType(FloatType(32, saturated), 2), "c"), Field(FixedLengthArrayType(FloatType(32, saturated), 7), "d"), PaddingField(VoidType(3)), diff --git a/pydsdl/_serializable/_primitive.py b/pydsdl/_serializable/_primitive.py index e227e15..8644867 100644 --- a/pydsdl/_serializable/_primitive.py +++ b/pydsdl/_serializable/_primitive.py @@ -10,7 +10,7 @@ import typing import fractions from .._bit_length_set import BitLengthSet -from ._serializable import SerializableType, TypeParameterError +from ._serializable import SerializableType, TypeParameterError, AggregationFailure ValueRange = typing.NamedTuple("ValueRange", [("min", fractions.Fraction), ("max", fractions.Fraction)]) @@ -51,6 +51,14 @@ def __init__(self, bit_length: int, cast_mode: "PrimitiveType.CastMode"): def bit_length_set(self) -> BitLengthSet: return BitLengthSet(self.bit_length) + @property + def deprecated(self) -> bool: + """Primitive types cannot be deprecated.""" + return False + + def _check_aggregation(self, aggregate: "SerializableType") -> typing.Optional[AggregationFailure]: + return super()._check_aggregation(aggregate) + @property def bit_length(self) -> int: """ @@ -96,14 +104,11 @@ def __repr__(self) -> str: class BooleanType(PrimitiveType): - def __init__(self, cast_mode: PrimitiveType.CastMode): - super().__init__(bit_length=1, cast_mode=cast_mode) - - if cast_mode != PrimitiveType.CastMode.SATURATED: - raise InvalidCastModeError("Invalid cast mode for boolean: %r" % cast_mode) + def __init__(self) -> None: + super().__init__(bit_length=1, cast_mode=PrimitiveType.CastMode.SATURATED) def __str__(self) -> str: - return self._cast_mode_name + " bool" + return "bool" class ArithmeticType(PrimitiveType): @@ -165,6 +170,46 @@ def __str__(self) -> str: return self._cast_mode_name + " uint" + str(self.bit_length) +class ByteType(UnsignedIntegerType): + """ + This type is used as the array element type for byte strings. + """ + + def __init__(self) -> None: + super().__init__(bit_length=PrimitiveType.BITS_IN_BYTE, cast_mode=PrimitiveType.CastMode.TRUNCATED) + + def _check_aggregation(self, aggregate: "SerializableType") -> typing.Optional[AggregationFailure]: + from ._array import ArrayType + + if not isinstance(aggregate, ArrayType): + return AggregationFailure(self, aggregate, "The byte type can only be used as an array element type") + return super()._check_aggregation(aggregate) + + def __str__(self) -> str: + return "byte" + + +class UTF8Type(UnsignedIntegerType): + """ + This type is used as the array element type for UTF-8 strings. + """ + + def __init__(self) -> None: + super().__init__(bit_length=8, cast_mode=PrimitiveType.CastMode.TRUNCATED) + + def _check_aggregation(self, aggregate: "SerializableType") -> typing.Optional[AggregationFailure]: + from ._array import VariableLengthArrayType + + if not isinstance(aggregate, VariableLengthArrayType): + return AggregationFailure( + self, aggregate, "The utf8 type can only be used as a variable-length array element type" + ) + return super()._check_aggregation(aggregate) + + def __str__(self) -> str: + return "utf8" + + class FloatType(ArithmeticType): def __init__(self, bit_length: int, cast_mode: PrimitiveType.CastMode): super().__init__(bit_length, cast_mode) @@ -193,7 +238,17 @@ def __str__(self) -> str: def _unittest_primitive() -> None: from pytest import raises, approx - assert str(BooleanType(PrimitiveType.CastMode.SATURATED)) == "saturated bool" + assert str(BooleanType()) == "bool" + + assert str(ByteType()) == "byte" + assert ByteType().bit_length_set == {8} + assert ByteType().inclusive_value_range == (0, 255) # type: ignore + assert ByteType().cast_mode == PrimitiveType.CastMode.TRUNCATED + + assert str(UTF8Type()) == "utf8" + assert UTF8Type().bit_length_set == {8} + assert UTF8Type().inclusive_value_range == (0, 255) # type: ignore + assert UTF8Type().cast_mode == PrimitiveType.CastMode.TRUNCATED assert str(SignedIntegerType(15, PrimitiveType.CastMode.SATURATED)) == "saturated int15" assert SignedIntegerType(64, PrimitiveType.CastMode.SATURATED).bit_length_set == {64} @@ -237,18 +292,18 @@ def _unittest_primitive() -> None: ) a = UnsignedIntegerType(2, PrimitiveType.CastMode.TRUNCATED) - b = BooleanType(PrimitiveType.CastMode.SATURATED) + b = BooleanType() assert hash(a) != hash(b) assert hash(a) == hash(UnsignedIntegerType(2, PrimitiveType.CastMode.TRUNCATED)) assert a == UnsignedIntegerType(2, PrimitiveType.CastMode.TRUNCATED) assert b != UnsignedIntegerType(2, PrimitiveType.CastMode.TRUNCATED) assert a != b - assert b == BooleanType(PrimitiveType.CastMode.SATURATED) + assert b == BooleanType() assert b != 123 # Not implemented for bl in range(1, PrimitiveType.MAX_BIT_LENGTH + 1): if bl > 1: t = UnsignedIntegerType(bl, PrimitiveType.CastMode.SATURATED) # type: PrimitiveType else: - t = BooleanType(PrimitiveType.CastMode.SATURATED) + t = BooleanType() assert t.standard_bit_length == (t.bit_length in {8, 16, 32, 64, 128, 256}) diff --git a/pydsdl/_serializable/_serializable.py b/pydsdl/_serializable/_serializable.py index 6065cf5..67a0416 100644 --- a/pydsdl/_serializable/_serializable.py +++ b/pydsdl/_serializable/_serializable.py @@ -3,6 +3,7 @@ # Author: Pavel Kirienko import abc +from typing import Optional, NamedTuple from .. import _expression from .. import _error from .._bit_length_set import BitLengthSet @@ -12,6 +13,20 @@ class TypeParameterError(_error.InvalidDefinitionError): pass +AggregationFailure = NamedTuple( + "AggregationFailure", + [ + ("inner", "SerializableType"), + ("outer", "SerializableType"), + ("message", str), + ], +) +""" +This is returned by :meth:`SerializableType._check_aggregation()` to indicate the reason of the failure. +Eventually this will need to be replaced with a dataclass (sadly no dataclasses in Python 3.6). +""" + + class SerializableType(_expression.Any): """ Instances are immutable. @@ -56,6 +71,30 @@ def alignment_requirement(self) -> int: """ raise NotImplementedError + @property + @abc.abstractmethod + def deprecated(self) -> bool: + """ + Deprecation is transitive. + This property is used to propagate deprecation information from the type to its aggregates. + """ + raise NotImplementedError + + def _check_aggregation(self, aggregate: "SerializableType") -> Optional[AggregationFailure]: + """ + Returns None iff the argument is a suitable aggregate type for this element type; + otherwise, returns an instance of :class:`AggregationFailure` describing the reason of the failure. + This is needed to detect incorrect aggregations, such as padding fields in unions, + or standalone utf8 or byte, or a deprecated type nested into a non-deprecated type, etc. + """ + if self.deprecated and not aggregate.deprecated: + return AggregationFailure( + self, + aggregate, + "A non-deprecated type %s cannot depend on a deprecated type %s" % (aggregate, self), + ) + return None + def _attribute(self, name: _expression.String) -> _expression.Any: if name.native_value == "_bit_length_": # Experimental non-standard extension try: diff --git a/pydsdl/_serializable/_void.py b/pydsdl/_serializable/_void.py index 576ba2a..7c9e1c1 100644 --- a/pydsdl/_serializable/_void.py +++ b/pydsdl/_serializable/_void.py @@ -2,8 +2,9 @@ # This software is distributed under the terms of the MIT License. # Author: Pavel Kirienko +import typing from .._bit_length_set import BitLengthSet -from ._serializable import SerializableType +from ._serializable import SerializableType, AggregationFailure from ._primitive import InvalidBitLengthError @@ -24,6 +25,18 @@ def __init__(self, bit_length: int): def bit_length_set(self) -> BitLengthSet: return BitLengthSet(self.bit_length) + @property + def deprecated(self) -> bool: + """Void types cannot be deprecated.""" + return False + + def _check_aggregation(self, aggregate: "SerializableType") -> typing.Optional[AggregationFailure]: + from ._composite import StructureType, CompositeType + + if not isinstance(aggregate, CompositeType) or not isinstance(aggregate.inner_type, StructureType): + return AggregationFailure(self, aggregate, "Void types can only be aggregated into structures") + return super()._check_aggregation(aggregate) + @property def bit_length(self) -> int: """ diff --git a/pydsdl/_test.py b/pydsdl/_test.py index 83eb14d..7b02713 100644 --- a/pydsdl/_test.py +++ b/pydsdl/_test.py @@ -7,11 +7,11 @@ from __future__ import annotations import tempfile -from typing import Sequence, Type, Iterable +from typing import Sequence, Type, Iterable, Any from pathlib import Path from textwrap import dedent import pytest # This is only safe to import in test files! -from . import _data_type_builder +from . import InvalidDefinitionError from . import _expression from . import _error from . import _parser @@ -67,13 +67,16 @@ def is_fs_case_sensitive(self) -> bool: def parse_definition( - definition: _dsdl_definition.DSDLDefinition, lookup_definitions: Sequence[_dsdl_definition.DSDLDefinition] + definition: _dsdl_definition.DSDLDefinition, + lookup_definitions: Sequence[_dsdl_definition.DSDLDefinition], + **kwargs: Any, ) -> _serializable.CompositeType: return definition.read( lookup_definitions, [], print_output_handler=lambda line, text: print("Output from line %d:" % line, text), allow_unregulated_fixed_port_id=False, + **kwargs, ) @@ -281,7 +284,7 @@ def _unittest_simple(wrkspc: Workspace) -> None: truncated float16 PI = 3.1415926535897932384626433 uint8 a vendor.nested.Empty.255.255[5] b - saturated bool [ <= 255 ] c + bool [ <= 255 ] c """ ), ) @@ -309,7 +312,7 @@ def _unittest_simple(wrkspc: Workspace) -> None: assert len(p.fields) == 3 assert str(p.fields[0]) == "saturated uint8 a" assert str(p.fields[1]) == "vendor.nested.Empty.255.255[5] b" - assert str(p.fields[2]) == "saturated bool[<=255] c" + assert str(p.fields[2]) == "bool[<=255] c" def _unittest_comments(wrkspc: Workspace) -> None: @@ -408,7 +411,7 @@ def _unittest_comments(wrkspc: Workspace) -> None: truncated float16 PI = 3.1415926535897932384626433 uint8 a vendor.nested.Empty.255.255[5] b - saturated bool [ <= 255 ] c + bool [ <= 255 ] c """ ), ) @@ -535,9 +538,15 @@ def standalone(rel_path: str, definition: str, allow_unregulated: bool = False) with raises(_parser.DSDLSyntaxError): standalone("vendor/types/A.1.0.dsdl", "truncated uavcan.node.Heartbeat.1.0 field\n@sealed") - with raises(_serializable._primitive.InvalidCastModeError): + with raises(_parser.DSDLSyntaxError): standalone("vendor/types/A.1.0.dsdl", "truncated bool foo\n@sealed") + with raises(_parser.DSDLSyntaxError): + standalone("vendor/types/A.1.0.dsdl", "saturated utf8 foo\n@sealed") + + with raises(_parser.DSDLSyntaxError): + standalone("vendor/types/A.1.0.dsdl", "saturated byte foo\n@sealed") + with raises(_serializable._primitive.InvalidCastModeError): standalone("vendor/types/A.1.0.dsdl", "truncated int8 foo\n@sealed") @@ -753,6 +762,21 @@ def standalone(rel_path: str, definition: str, allow_unregulated: bool = False) ), ) + with raises(_error.InvalidDefinitionError, match="(?i).*not a valid field type.*"): + standalone("vendor/types/A.1.0.dsdl", "utf8 foo\n@sealed") + + with raises(_error.InvalidDefinitionError, match="(?i).*element type.*"): + standalone("vendor/types/A.1.0.dsdl", "utf8[10] foo\n@sealed") # Cannot be fixed-length. + + with raises(_error.InvalidDefinitionError, match="(?i).*not a valid field type.*"): + standalone("vendor/types/A.1.0.dsdl", "byte foo\n@sealed") + + with raises(_error.InvalidDefinitionError, match="(?i).*not a valid field type.*"): + standalone("vendor/types/A.1.0.dsdl", "utf8 FOO = 1\n@sealed") + + with raises(_error.InvalidDefinitionError, match="(?i).*not a valid field type.*"): + standalone("vendor/types/A.1.0.dsdl", "byte FOO = 1\n@sealed") + def _unittest_print(wrkspc: Workspace) -> None: printed_items = None # type: tuple[int, str] | None @@ -1727,7 +1751,7 @@ def _unittest_inconsistent_deprecation(wrkspc: Workspace) -> None: ], ) - with raises(_error.InvalidDefinitionError, match="(?i).*depend.*deprecated.*"): + with raises(_error.InvalidDefinitionError, match="(?i).*depend.*deprecated.*") as exc_info: parse_definition( wrkspc.parse_new( "ns/C.1.0.dsdl", @@ -1740,6 +1764,22 @@ def _unittest_inconsistent_deprecation(wrkspc: Workspace) -> None: ), [wrkspc.parse_new("ns/X.1.0.dsdl", "@deprecated\n@sealed")], ) + print(exc_info.value.text) + + with raises(_error.InvalidDefinitionError, match="(?i).*depend.*deprecated.*") as exc_info: + parse_definition( + wrkspc.parse_new( + "ns/C.1.0.dsdl", + dedent( + """ + X.1.0[<9] b # Ensure the deprecation property is transitive. + @sealed + """ + ), + ), + [wrkspc.parse_new("ns/X.1.0.dsdl", "@deprecated\n@sealed")], + ) + print(exc_info.value.text) parse_definition( wrkspc.parse_new( @@ -1913,6 +1953,42 @@ def _unittest_dsdl_parser_basics(wrkspc: Workspace) -> None: ) +def _unittest_dsdl_parser_utf8_bytes(wrkspc: Workspace) -> None: + ty = parse_definition( + wrkspc.parse_new( + "ns/A.1.0.dsdl", + dedent( + r""" + byte[10] bytes_fixed + byte[<=10] bytes_variable + utf8[<=10] string + @extent 256 * 8 + """ + ), + ), + [], + ) + t = ty.fields[0].data_type + assert isinstance(t, _serializable.FixedLengthArrayType) + assert isinstance(t.element_type, _serializable.ByteType) + assert isinstance(t.element_type, _serializable.UnsignedIntegerType) + assert t.capacity == 10 + assert not t.string_like + + t = ty.fields[1].data_type + assert isinstance(t, _serializable.VariableLengthArrayType) + assert isinstance(t.element_type, _serializable.ByteType) + assert isinstance(t.element_type, _serializable.UnsignedIntegerType) + assert t.capacity == 10 + + t = ty.fields[2].data_type + assert isinstance(t, _serializable.VariableLengthArrayType) + assert isinstance(t.element_type, _serializable.UTF8Type) + assert isinstance(t.element_type, _serializable.UnsignedIntegerType) + assert t.capacity == 10 + assert t.string_like + + def _unittest_dsdl_parser_expressions(wrkspc: Workspace) -> None: from pytest import raises diff --git a/pydsdl/grammar.parsimonious b/pydsdl/grammar.parsimonious index d56cb25..27bdcd4 100644 --- a/pydsdl/grammar.parsimonious +++ b/pydsdl/grammar.parsimonious @@ -47,18 +47,22 @@ type_scalar = type_versioned type_versioned = identifier ("." identifier)* "." type_version_specifier type_version_specifier = literal_integer_decimal "." literal_integer_decimal -type_primitive = type_primitive_truncated +type_primitive = type_primitive_boolean + / type_primitive_byte + / type_primitive_utf8 + / type_primitive_truncated / type_primitive_saturated +type_primitive_boolean = "bool" +type_primitive_byte = "byte" +type_primitive_utf8 = "utf8" type_primitive_truncated = "truncated" _ type_primitive_name type_primitive_saturated = ("saturated" _)? type_primitive_name # Defaults to this. -type_primitive_name = type_primitive_name_boolean - / type_primitive_name_unsigned_integer +type_primitive_name = type_primitive_name_unsigned_integer / type_primitive_name_signed_integer / type_primitive_name_floating_point -type_primitive_name_boolean = "bool" type_primitive_name_unsigned_integer = "uint" type_bit_length_suffix type_primitive_name_signed_integer = "int" type_bit_length_suffix type_primitive_name_floating_point = "float" type_bit_length_suffix diff --git a/setup.cfg b/setup.cfg index 8d934ba..e52a01c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -45,6 +45,8 @@ log_level = DEBUG log_cli = true log_cli_level = WARNING addopts = --doctest-modules -v -p no:unraisableexception +filterwarnings = + ignore:.*string_like.*:DeprecationWarning # -------------------------------------------------- MYPY -------------------------------------------------- [mypy] @@ -117,7 +119,9 @@ disable= too-many-public-methods, consider-using-f-string, unspecified-encoding, - use-implicit-booleaness-not-comparison + use-implicit-booleaness-not-comparison, + too-many-arguments, + too-many-locals [pylint.REPORTS] output-format=colorized