diff --git a/benchmarks/benchmarks.py b/benchmarks/benchmarks.py index 07b236075..4f921d20c 100644 --- a/benchmarks/benchmarks.py +++ b/benchmarks/benchmarks.py @@ -6,32 +6,32 @@ @dataclass class TestMessage(betterproto.Message): - foo: int = betterproto.uint32_field(0) - bar: str = betterproto.string_field(1) - baz: float = betterproto.float_field(2) + foo: int = betterproto.uint32_field(1) + bar: str = betterproto.string_field(2) + baz: float = betterproto.float_field(3) @dataclass class TestNestedChildMessage(betterproto.Message): - str_key: str = betterproto.string_field(0) - bytes_key: bytes = betterproto.bytes_field(1) - bool_key: bool = betterproto.bool_field(2) - float_key: float = betterproto.float_field(3) - int_key: int = betterproto.uint64_field(4) + str_key: str = betterproto.string_field(1) + bytes_key: bytes = betterproto.bytes_field(2) + bool_key: bool = betterproto.bool_field(3) + float_key: float = betterproto.float_field(4) + int_key: int = betterproto.uint64_field(5) @dataclass class TestNestedMessage(betterproto.Message): - foo: TestNestedChildMessage = betterproto.message_field(0) - bar: TestNestedChildMessage = betterproto.message_field(1) - baz: TestNestedChildMessage = betterproto.message_field(2) + foo: TestNestedChildMessage = betterproto.message_field(1) + bar: TestNestedChildMessage = betterproto.message_field(2) + baz: TestNestedChildMessage = betterproto.message_field(3) @dataclass class TestRepeatedMessage(betterproto.Message): - foo_repeat: List[str] = betterproto.string_field(0) - bar_repeat: List[int] = betterproto.int64_field(1) - baz_repeat: List[bool] = betterproto.bool_field(2) + foo_repeat: List[str] = betterproto.string_field(1) + bar_repeat: List[int] = betterproto.int64_field(2) + baz_repeat: List[bool] = betterproto.bool_field(3) class BenchMessage: @@ -44,25 +44,14 @@ def setup(self): self.instance_filled_bytes = bytes(self.instance_filled) self.instance_filled_nested = TestNestedMessage( TestNestedChildMessage("foo", bytearray(b"test1"), True, 0.1234, 500), - TestNestedChildMessage("bar", bytearray(b"test2"), True, 3.1415, -302), + TestNestedChildMessage("bar", bytearray(b"test2"), True, 3.1415, 302), TestNestedChildMessage("baz", bytearray(b"test3"), False, 1e5, 300), ) self.instance_filled_nested_bytes = bytes(self.instance_filled_nested) self.instance_filled_repeated = TestRepeatedMessage( - [ - "test1", - "test2", - "test3", - "test4", - "test5", - "test6", - "test7", - "test8", - "test9", - "test10", - ], - [2, -100, 0, 500000, 600, -425678, 1000000000, -300, 1, -694214214466], - [True, False, False, False, True, True, False, True, False, False], + [f"test{i}" for i in range(1_000)], + [(i-500)**3 for i in range(1_000)], + [i%2==0 for i in range(1_000)], ) self.instance_filled_repeated_bytes = bytes(self.instance_filled_repeated) @@ -71,9 +60,9 @@ def time_overhead(self): @dataclass class Message(betterproto.Message): - foo: int = betterproto.uint32_field(0) - bar: str = betterproto.string_field(1) - baz: float = betterproto.float_field(2) + foo: int = betterproto.uint32_field(1) + bar: str = betterproto.string_field(2) + baz: float = betterproto.float_field(3) def time_instantiation(self): """Time instantiation""" diff --git a/betterproto-extras/.gitignore b/betterproto-extras/.gitignore new file mode 100644 index 000000000..af3ca5ef1 --- /dev/null +++ b/betterproto-extras/.gitignore @@ -0,0 +1,72 @@ +/target + +# Byte-compiled / optimized / DLL files +__pycache__/ +.pytest_cache/ +*.py[cod] + +# C extensions +*.so + +# Distribution / packaging +.Python +.venv/ +env/ +bin/ +build/ +develop-eggs/ +dist/ +eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +include/ +man/ +venv/ +*.egg-info/ +.installed.cfg +*.egg + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt +pip-selfcheck.json + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.cache +nosetests.xml +coverage.xml + +# Translations +*.mo + +# Mr Developer +.mr.developer.cfg +.project +.pydevproject + +# Rope +.ropeproject + +# Django stuff: +*.log +*.pot + +.DS_Store + +# Sphinx documentation +docs/_build/ + +# PyCharm +.idea/ + +# VSCode +.vscode/ + +# Pyenv +.python-version \ No newline at end of file diff --git a/betterproto-extras/Cargo.lock b/betterproto-extras/Cargo.lock new file mode 100644 index 000000000..dffb5145f --- /dev/null +++ b/betterproto-extras/Cargo.lock @@ -0,0 +1,363 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "anyhow" +version = "1.0.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "betterproto-extras" +version = "0.1.0" +dependencies = [ + "indoc 2.0.4", + "prost", + "pyo3", + "thiserror", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bytes" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "either" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" + +[[package]] +name = "indoc" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa799dd5ed20a7e349f3b4639aa80d74549c81716d9ec4f994c9b5815598306" + +[[package]] +name = "indoc" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e186cfbae8084e513daff4240b4797e342f988cecda4fb6c939150f96315fd8" + +[[package]] +name = "itertools" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + +[[package]] +name = "libc" +version = "0.2.148" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b" + +[[package]] +name = "lock_api" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "memoffset" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" + +[[package]] +name = "parking_lot" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets", +] + +[[package]] +name = "proc-macro2" +version = "1.0.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d433d9f1a3e8c1263d9456598b16fec66f4acc9a74dacffd35c7bb09b3a1328" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4fdd22f3b9c31b53c060df4a0613a1c7f062d4115a2b984dd15b1858f7e340d" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "265baba7fabd416cf5078179f7d2cbeca4ce7a9041111900675ea7c4cb8a4c32" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 2.0.36", +] + +[[package]] +name = "pyo3" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e681a6cfdc4adcc93b4d3cf993749a4552018ee0a9b65fc0ccfad74352c72a38" +dependencies = [ + "cfg-if", + "indoc 1.0.9", + "libc", + "memoffset", + "parking_lot", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", + "unindent", +] + +[[package]] +name = "pyo3-build-config" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "076c73d0bc438f7a4ef6fdd0c3bb4732149136abd952b110ac93e4edb13a6ba5" +dependencies = [ + "once_cell", + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e53cee42e77ebe256066ba8aa77eff722b3bb91f3419177cf4cd0f304d3284d9" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfeb4c99597e136528c6dd7d5e3de5434d1ceaf487436a3f03b2d56b6fc9efd1" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "947dc12175c254889edc0c02e399476c2f652b4b9ebd123aa655c224de259536" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "quote" +version = "1.0.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "redox_syscall" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +dependencies = [ + "bitflags", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "smallvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e02e55d62894af2a08aca894c6577281f76769ba47c94d5756bec8ac6e7373" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "target-lexicon" +version = "0.12.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d0e916b1148c8e263850e1ebcbd046f333e0683c724876bb0da63ea4373dc8a" + +[[package]] +name = "thiserror" +version = "1.0.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d6d7a740b8a666a7e828dd00da9c0dc290dff53154ea77ac109281de90589b7" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.36", +] + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "unindent" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1766d682d402817b5ac4490b3c3002d91dfa0d22812f341609f97b08757359c" + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" diff --git a/betterproto-extras/Cargo.toml b/betterproto-extras/Cargo.toml new file mode 100644 index 000000000..126f460cb --- /dev/null +++ b/betterproto-extras/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "betterproto-extras" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[lib] +name = "betterproto_extras" +crate-type = ["cdylib"] + +[dependencies] +indoc = "2.0.3" +prost = { version = "0.12.1", features = ["no-recursion-limit"] } +pyo3 = { version = "0.19.0", features = ["abi3-py37", "extension-module"] } +thiserror = "1.0.48" diff --git a/betterproto-extras/betterproto_extras.pyi b/betterproto-extras/betterproto_extras.pyi new file mode 100644 index 000000000..83a51781c --- /dev/null +++ b/betterproto-extras/betterproto_extras.pyi @@ -0,0 +1,10 @@ +def deserialize(msg, data: bytes): + """ + Parses the binary encoded Protobuf `data` with respect to the metadata + given by the betterproto message `msg`, and merges the result into `msg`. + """ + +def serialize(msg) -> bytes: + """ + Get the binary encoded Protobuf representation of this message instance. + """ \ No newline at end of file diff --git a/betterproto-extras/pyproject.toml b/betterproto-extras/pyproject.toml new file mode 100644 index 000000000..e73b0e216 --- /dev/null +++ b/betterproto-extras/pyproject.toml @@ -0,0 +1,16 @@ +[build-system] +requires = ["maturin>=1.2,<2.0"] +build-backend = "maturin" + +[project] +name = "betterproto-extras" +requires-python = ">=3.7" +classifiers = [ + "Programming Language :: Rust", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", +] + + +[tool.maturin] +features = ["pyo3/extension-module"] diff --git a/betterproto-extras/src/betterproto_interop/error.rs b/betterproto-extras/src/betterproto_interop/error.rs new file mode 100644 index 000000000..42a00af3e --- /dev/null +++ b/betterproto-extras/src/betterproto_interop/error.rs @@ -0,0 +1,18 @@ +use pyo3::PyErr; +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum InteropError { + #[error("Given object is not a valid betterproto message.")] + NoBetterprotoMessage(#[from] PyErr), + #[error("Unsupported value type `{0}`.")] + UnsupportedValueType(String), + #[error("Unsupported key type `{0:?}`.")] + UnsupportedKeyType(String), + #[error("Unsupported wrapped type `{0:?}`.")] + UnsupportedWrappedType(String), + #[error("Given object is not a valid betterproto message.")] + IncompleteMetadata, +} + +pub type InteropResult = Result; diff --git a/betterproto-extras/src/betterproto_interop/field_meta.rs b/betterproto-extras/src/betterproto_interop/field_meta.rs new file mode 100644 index 000000000..d67a0cb7c --- /dev/null +++ b/betterproto-extras/src/betterproto_interop/field_meta.rs @@ -0,0 +1,136 @@ +use super::{ + error::{InteropError, InteropResult}, + message_class::BetterprotoMessageClass, + message_meta::BetterprotoMessageMeta, + BetterprotoEnumClass, +}; +use crate::{ + descriptors::{FieldAttribute, FieldDescriptor, ProtoType}, + Str, +}; +use pyo3::{FromPyObject, IntoPy, Python}; + +#[derive(FromPyObject)] +pub struct BetterprotoFieldMeta { + pub number: u32, + pub map_types: Option<(String, String)>, + pub proto_type: String, + pub wraps: Option, + pub optional: bool, +} + +impl BetterprotoFieldMeta { + pub fn into_descriptor( + self, + py: Python, + field_name: Str, + msg_meta: &BetterprotoMessageMeta, + ) -> InteropResult { + if let Some((key_type, value_type)) = self.map_types { + let key_type = convert_key_type(&key_type)?; + let value_type = + convert_value_type(py, &value_type, &format!("{field_name}.value"), msg_meta)?; + return Ok(FieldDescriptor { + name: field_name, + attribute: FieldAttribute::Map(key_type), + value_type, + }); + } + + let value_type = match self.wraps { + Some(wrapped_type) => convert_wrapped_type(&wrapped_type)?, + None => convert_value_type(py, &self.proto_type, &field_name, msg_meta)?, + }; + + let attribute = if self.optional { + FieldAttribute::Optional + } else if let Some(group) = msg_meta.oneof_group_by_field.get(field_name.as_ref()) { + FieldAttribute::Group(Str::from(group.as_ref())) + } else if msg_meta.is_list_field(&field_name)? { + FieldAttribute::Repeated + } else { + FieldAttribute::None + }; + + Ok(FieldDescriptor { + name: field_name, + value_type, + attribute, + }) + } +} + +fn convert_value_type( + py: Python, + type_name: &str, + field_name: &str, + msg_meta: &BetterprotoMessageMeta, +) -> InteropResult { + match type_name { + "bool" => Ok(ProtoType::Bool), + "int32" => Ok(ProtoType::Int32), + "int64" => Ok(ProtoType::Int64), + "uint32" => Ok(ProtoType::Uint32), + "uint64" => Ok(ProtoType::Uint64), + "sint32" => Ok(ProtoType::Sint32), + "sint64" => Ok(ProtoType::Sint64), + "float" => Ok(ProtoType::Float), + "double" => Ok(ProtoType::Double), + "fixed32" => Ok(ProtoType::Fixed32), + "sfixed32" => Ok(ProtoType::Sfixed32), + "fixed64" => Ok(ProtoType::Fixed64), + "sfixed64" => Ok(ProtoType::Sfixed64), + "string" => Ok(ProtoType::String), + "bytes" => Ok(ProtoType::Bytes), + "enum" => Ok(ProtoType::Enum(BetterprotoEnumClass( + msg_meta.get_class(field_name)?.into_py(py), + ))), + "message" => { + let cls = msg_meta.get_class(field_name)?; + if cls.getattr("__module__")?.extract::<&str>()? == "datetime" { + match cls.name()? { + "datetime" => return Ok(ProtoType::Timestamp), + "timedelta" => return Ok(ProtoType::Duration), + _ => {} + } + } + Ok(ProtoType::CustomMessage(BetterprotoMessageClass( + cls.into_py(py), + ))) + } + _ => Err(InteropError::UnsupportedValueType(type_name.to_string())), + } +} + +fn convert_key_type(type_name: &str) -> InteropResult { + match type_name { + "bool" => Ok(ProtoType::Bool), + "int32" => Ok(ProtoType::Int32), + "int64" => Ok(ProtoType::Int64), + "uint32" => Ok(ProtoType::Uint32), + "uint64" => Ok(ProtoType::Uint64), + "sint32" => Ok(ProtoType::Sint32), + "sint64" => Ok(ProtoType::Sint64), + "fixed32" => Ok(ProtoType::Fixed32), + "sfixed32" => Ok(ProtoType::Sfixed32), + "fixed64" => Ok(ProtoType::Fixed64), + "sfixed64" => Ok(ProtoType::Sfixed64), + "string" => Ok(ProtoType::String), + _ => Err(InteropError::UnsupportedKeyType(type_name.to_string())), + } +} + +fn convert_wrapped_type(type_name: &str) -> InteropResult { + match type_name { + "bool" => Ok(ProtoType::BoolValue), + "int32" => Ok(ProtoType::Int32Value), + "int64" => Ok(ProtoType::Int64Value), + "uint32" => Ok(ProtoType::UInt32Value), + "uint64" => Ok(ProtoType::UInt64Value), + "float" => Ok(ProtoType::FloatValue), + "double" => Ok(ProtoType::DoubleValue), + "string" => Ok(ProtoType::StringValue), + "bytes" => Ok(ProtoType::BytesValue), + _ => Err(InteropError::UnsupportedWrappedType(type_name.to_string())), + } +} diff --git a/betterproto-extras/src/betterproto_interop/message.rs b/betterproto-extras/src/betterproto_interop/message.rs new file mode 100644 index 000000000..f3a1d44e5 --- /dev/null +++ b/betterproto-extras/src/betterproto_interop/message.rs @@ -0,0 +1,94 @@ +use super::{error::InteropResult, BetterprotoMessageClass}; +use indoc::indoc; +use pyo3::{ + intern, + sync::GILOnceCell, + types::{PyBytes, PyModule}, + FromPyObject, IntoPy, PyAny, PyObject, Python, ToPyObject, +}; + +#[derive(FromPyObject, Clone, Copy)] +pub struct BetterprotoMessage<'py>(pub(super) &'py PyAny); + +impl<'py> BetterprotoMessage<'py> { + pub fn class(&self) -> BetterprotoMessageClass { + BetterprotoMessageClass(self.0.get_type().into_py(self.0.py())) + } + + pub fn py(&self) -> Python { + self.0.py() + } + + pub fn set_field(&self, field_name: &str, value: impl ToPyObject) -> InteropResult<()> { + self.0.setattr(field_name, value)?; + Ok(()) + } + + pub fn get_field(&'py self, field_name: &str) -> InteropResult> { + let py = self.py(); + static GETTER_CACHE: GILOnceCell = GILOnceCell::new(); + let getter = GETTER_CACHE + .get_or_init(py, || { + PyModule::from_code( + py, + indoc! {" + from betterproto import PLACEHOLDER + + def getter(msg, field_name): + value = msg._Message__raw_get(field_name) + if value is PLACEHOLDER: + return + return value + "}, + "", + "", + ) + .expect("This is a valid Python module") + .getattr("getter") + .expect("Attribute exists") + .to_object(py) + }) + .as_ref(py); + + let res = getter.call1((self.0, field_name))?.extract()?; + Ok(res) + } + + pub fn append_unknown_fields(&self, mut data: Vec) -> InteropResult<()> { + let attr_name = intern!(self.py(), "_unknown_fields"); + if !data.is_empty() { + let mut unknown_fields = self.0.getattr(attr_name)?.extract::>()?; + unknown_fields.append(&mut data); + self.0 + .setattr(attr_name, PyBytes::new(self.py(), &unknown_fields))?; + } + Ok(()) + } + + pub fn get_unknown_fields(&self) -> InteropResult> { + Ok(self + .0 + .getattr(intern!(self.py(), "_unknown_fields"))? + .extract()?) + } + + pub fn set_deserialized(&self) -> InteropResult<()> { + self.0 + .setattr(intern!(self.py(), "_serialized_on_wire"), true)?; + Ok(()) + } + + pub fn should_be_serialized(&self) -> InteropResult { + let res = self + .0 + .getattr(intern!(self.py(), "_serialized_on_wire"))? + .extract()?; + Ok(res) + } +} + +impl ToPyObject for BetterprotoMessage<'_> { + fn to_object(&self, py: Python<'_>) -> pyo3::PyObject { + self.0.to_object(py) + } +} diff --git a/betterproto-extras/src/betterproto_interop/message_class.rs b/betterproto-extras/src/betterproto_interop/message_class.rs new file mode 100644 index 000000000..1830758f7 --- /dev/null +++ b/betterproto-extras/src/betterproto_interop/message_class.rs @@ -0,0 +1,38 @@ +use super::{ + error::InteropResult, message::BetterprotoMessage, message_meta::BetterprotoMessageMeta, +}; +use crate::descriptors::MessageDescriptor; +use pyo3::{intern, pyclass, types::PyType, FromPyObject, Py, PyCell, Python}; + +#[derive(FromPyObject, Debug)] +pub struct BetterprotoMessageClass(pub(super) Py); + +impl BetterprotoMessageClass { + pub fn create_instance<'py>( + &'py self, + py: Python<'py>, + ) -> InteropResult> { + Ok(BetterprotoMessage(self.0.as_ref(py).call0()?)) + } + + pub fn descriptor<'py>(&'py self, py: Python<'py>) -> InteropResult<&'py MessageDescriptor> { + let cls = self.0.as_ref(py); + if let Ok(attr) = cls.getattr(intern!(py, "_betterproto_extras")) { + if let Ok(cell) = attr.downcast::>() { + return Ok(&cell.get().0); + } + } + + let desc = cls + .call0()? + .getattr(intern!(py, "_betterproto"))? + .extract::()? + .into_descriptor(py)?; + let cell = PyCell::new(py, DescriptorWrapper(desc))?; + cls.setattr(intern!(py, "_betterproto_extras"), cell)?; + Ok(&cell.get().0) + } +} + +#[pyclass(frozen)] +struct DescriptorWrapper(MessageDescriptor); diff --git a/betterproto-extras/src/betterproto_interop/message_meta.rs b/betterproto-extras/src/betterproto_interop/message_meta.rs new file mode 100644 index 000000000..c6430c3ea --- /dev/null +++ b/betterproto-extras/src/betterproto_interop/message_meta.rs @@ -0,0 +1,52 @@ +use std::collections::HashMap; + +use pyo3::{ + types::{PyDict, PyList, PyType}, + FromPyObject, PyAny, Python, +}; + +use crate::descriptors::MessageDescriptor; + +use super::{ + error::{InteropError, InteropResult}, + field_meta::BetterprotoFieldMeta, +}; + +#[derive(FromPyObject)] +pub struct BetterprotoMessageMeta<'py> { + pub cls_by_field: HashMap, + pub meta_by_field_name: &'py PyDict, + pub oneof_group_by_field: HashMap, + pub default_gen: HashMap, +} + +impl<'py> BetterprotoMessageMeta<'py> { + pub fn is_list_field(&self, field_name: &str) -> InteropResult { + let cls = self + .default_gen + .get(field_name) + .ok_or(InteropError::IncompleteMetadata)?; + Ok(cls.is(cls.py().get_type::())) + } + + pub fn get_class(&self, field_name: &str) -> InteropResult<&'py PyType> { + let cls = self + .cls_by_field + .get(field_name) + .ok_or(InteropError::IncompleteMetadata)?; + Ok(cls) + } + + pub fn into_descriptor(self, py: Python) -> InteropResult { + let fields = self + .meta_by_field_name + .iter() + .map(|(name, meta)| { + let name = name.extract::()?; + let meta = meta.extract::()?; + Ok((meta.number, meta.into_descriptor(py, name.into(), &self)?)) + }) + .collect::>>()?; + Ok(MessageDescriptor { fields }) + } +} diff --git a/betterproto-extras/src/betterproto_interop/mod.rs b/betterproto-extras/src/betterproto_interop/mod.rs new file mode 100644 index 000000000..09b015282 --- /dev/null +++ b/betterproto-extras/src/betterproto_interop/mod.rs @@ -0,0 +1,30 @@ +mod error; +mod field_meta; +mod message; +mod message_class; +mod message_meta; + +pub use self::{ + error::{InteropError, InteropResult}, + message::BetterprotoMessage, + message_class::BetterprotoMessageClass, +}; +use pyo3::{ + exceptions::PyValueError, types::PyType, FromPyObject, Py, PyObject, Python, ToPyObject, +}; + +#[derive(FromPyObject, Debug)] +pub struct BetterprotoEnumClass(Py); + +impl BetterprotoEnumClass { + pub fn create_instance(&self, py: Python, x: i32) -> InteropResult { + let res = self.0.call1(py, (x,)).or_else(|err| { + if err.is_instance_of::(py) { + Ok(x.to_object(py)) + } else { + Err(err) + } + })?; + Ok(res) + } +} diff --git a/betterproto-extras/src/decode/custom_message.rs b/betterproto-extras/src/decode/custom_message.rs new file mode 100644 index 000000000..942105400 --- /dev/null +++ b/betterproto-extras/src/decode/custom_message.rs @@ -0,0 +1,116 @@ +use super::{ + error::{DecodeError, DecodeResult}, + field::FieldBuilder, + MessageBuilder, +}; +use crate::{ + betterproto_interop::{BetterprotoMessage, InteropResult}, + descriptors::MessageDescriptor, + Str, +}; +use prost::{ + bytes::{Buf, Bytes}, + encoding::{self as enc, decode_key, WireType}, +}; +use pyo3::Python; +use std::collections::HashMap; + +pub struct CustomMessageBuilder<'a, 'py> { + fields: HashMap>, + active_groups: HashMap, + unknown_fields: Vec, +} + +impl<'a, 'py> CustomMessageBuilder<'a, 'py> { + pub fn new(py: Python<'py>, descriptor: &'a MessageDescriptor) -> Self { + Self { + fields: descriptor + .fields + .iter() + .map(|(tag, descriptor)| (*tag, FieldBuilder::new(py, descriptor))) + .collect(), + active_groups: HashMap::new(), + unknown_fields: Vec::new(), + } + } + + pub fn merge_into(self, msg: BetterprotoMessage) -> InteropResult<()> { + for (name, value) in self + .fields + .into_values() + .filter_map(|field| field.into_result()) + { + msg.set_field(name, value)?; + } + msg.append_unknown_fields(self.unknown_fields)?; + msg.set_deserialized()?; + Ok(()) + } + + pub fn parse_next_unknown( + &mut self, + tag: u32, + wire_type: WireType, + buf: &mut impl Buf, + ) -> DecodeResult<()> { + enc::encode_key(tag, wire_type, &mut self.unknown_fields); + match wire_type { + WireType::Varint => { + let value = enc::decode_varint(buf)?; + enc::encode_varint(value, &mut self.unknown_fields); + } + WireType::SixtyFourBit => { + let mut value = [0; 8]; + if buf.remaining() < value.len() { + return Err(DecodeError::InvalidData); + } + buf.copy_to_slice(&mut value); + self.unknown_fields.extend_from_slice(&value); + } + WireType::LengthDelimited => { + let mut value = Bytes::default(); + enc::bytes::merge(wire_type, &mut value, buf, Default::default())?; + enc::encode_varint(value.len() as u64, &mut self.unknown_fields); + self.unknown_fields.extend(value); + } + WireType::ThirtyTwoBit => { + let mut value = [0; 4]; + if buf.remaining() < value.len() { + return Err(DecodeError::InvalidData); + } + buf.copy_to_slice(&mut value); + self.unknown_fields.extend_from_slice(&value); + } + _ => return Err(DecodeError::InvalidData), + }; + + Ok(()) + } +} + +impl MessageBuilder for CustomMessageBuilder<'_, '_> { + fn parse_next_field(&mut self, buf: &mut impl Buf) -> DecodeResult<()> { + let (tag, wire_type) = decode_key(buf)?; + let group = match self.fields.get_mut(&tag) { + Some(builder) => { + builder.parse_next(wire_type, buf)?; + builder.group() + } + None => { + self.parse_next_unknown(tag, wire_type, buf)?; + None + } + }; + if let Some(group) = group { + if let Some(previous_tag) = self.active_groups.insert(group, tag) { + if previous_tag != tag { + self.fields + .get_mut(&previous_tag) + .expect("Field exists") + .reset() + } + } + } + Ok(()) + } +} diff --git a/betterproto-extras/src/decode/error.rs b/betterproto-extras/src/decode/error.rs new file mode 100644 index 000000000..720f45669 --- /dev/null +++ b/betterproto-extras/src/decode/error.rs @@ -0,0 +1,23 @@ +use crate::betterproto_interop::InteropError; +use pyo3::{exceptions::PyRuntimeError, PyErr}; +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum DecodeError { + #[error(transparent)] + Interop(#[from] InteropError), + #[error("The given binary data does not match the protobuf schema.")] + ProstDecode(#[from] prost::DecodeError), + #[error("The given binary data does not match the protobuf schema.")] + InvalidMapEntryTag, + #[error("The given binary data is not a valid protobuf message.")] + InvalidData, +} + +pub type DecodeResult = Result; + +impl From for PyErr { + fn from(value: DecodeError) -> Self { + PyRuntimeError::new_err(value.to_string()) + } +} diff --git a/betterproto-extras/src/decode/field.rs b/betterproto-extras/src/decode/field.rs new file mode 100644 index 000000000..5e22f8be8 --- /dev/null +++ b/betterproto-extras/src/decode/field.rs @@ -0,0 +1,49 @@ +use super::{error::DecodeResult, value::ValueBuilder}; +use crate::{ + descriptors::{FieldAttribute, FieldDescriptor}, + Str, +}; +use prost::{bytes::Buf, encoding::WireType}; +use pyo3::{PyObject, Python}; + +pub struct FieldBuilder<'a, 'py> { + descriptor: &'a FieldDescriptor, + value: ValueBuilder<'a, 'py>, +} + +impl<'a, 'py> FieldBuilder<'a, 'py> { + pub fn new(py: Python<'py>, descriptor: &'a FieldDescriptor) -> Self { + Self { + descriptor, + value: ValueBuilder::new(py, &descriptor.value_type), + } + } + + pub fn into_result(self) -> Option<(&'a str, PyObject)> { + self.value + .into_object() + .map(|obj| (self.descriptor.name.as_ref(), obj)) + } + + pub fn group(&self) -> Option { + match &self.descriptor.attribute { + FieldAttribute::Group(name) => Some(name.clone()), + _ => None, + } + } + + pub fn reset(&mut self) { + self.value.reset() + } + + pub fn parse_next(&mut self, wire_type: WireType, buf: &mut impl Buf) -> DecodeResult<()> { + match &self.descriptor.attribute { + FieldAttribute::Repeated => self.value.parse_next_list_entry(wire_type, buf)?, + FieldAttribute::Map(key_type) => { + self.value.parse_next_map_entry(wire_type, key_type, buf)? + } + _ => self.value.parse_next_single(wire_type, buf)?, + } + Ok(()) + } +} diff --git a/betterproto-extras/src/decode/map_entry.rs b/betterproto-extras/src/decode/map_entry.rs new file mode 100644 index 000000000..fb907ba48 --- /dev/null +++ b/betterproto-extras/src/decode/map_entry.rs @@ -0,0 +1,37 @@ +use super::{value::ValueBuilder, DecodeError, DecodeResult, MessageBuilder}; +use crate::descriptors::ProtoType; +use prost::{bytes::Buf, encoding::decode_key}; +use pyo3::{PyObject, Python}; + +pub struct MapEntryBuilder<'a, 'py> { + key: ValueBuilder<'a, 'py>, + value: ValueBuilder<'a, 'py>, +} + +impl<'a, 'py> MapEntryBuilder<'a, 'py> { + pub fn new(py: Python<'py>, key_type: &'a ProtoType, value_type: &'a ProtoType) -> Self { + Self { + key: ValueBuilder::new(py, key_type), + value: ValueBuilder::new(py, value_type), + } + } + + pub fn into_tuple(self) -> DecodeResult<(PyObject, PyObject)> { + Ok(( + self.key.into_object_or_default()?, + self.value.into_object_or_default()?, + )) + } +} + +impl MessageBuilder for MapEntryBuilder<'_, '_> { + fn parse_next_field(&mut self, buf: &mut impl Buf) -> DecodeResult<()> { + let (tag, wire_type) = decode_key(buf)?; + match tag { + 1 => self.key.parse_next_single(wire_type, buf)?, + 2 => self.value.parse_next_single(wire_type, buf)?, + _ => Err(DecodeError::InvalidMapEntryTag)?, + } + Ok(()) + } +} diff --git a/betterproto-extras/src/decode/mod.rs b/betterproto-extras/src/decode/mod.rs new file mode 100644 index 000000000..2ff69d1d4 --- /dev/null +++ b/betterproto-extras/src/decode/mod.rs @@ -0,0 +1,52 @@ +mod custom_message; +mod error; +mod field; +mod map_entry; +mod value; + +use self::custom_message::CustomMessageBuilder; +pub use self::error::{DecodeError, DecodeResult}; +use crate::betterproto_interop::BetterprotoMessage; +use prost::{ + bytes::Buf, + encoding::{check_wire_type, decode_varint, WireType}, +}; + +pub fn merge_into_message(msg: BetterprotoMessage, buf: &mut impl Buf) -> DecodeResult<()> { + let py = msg.py(); + let cls = msg.class(); + let mut builder = CustomMessageBuilder::new(py, cls.descriptor(py)?); + while buf.has_remaining() { + builder.parse_next_field(buf)?; + } + builder.merge_into(msg)?; + Ok(()) +} + +trait MessageBuilder { + fn parse_next_field(&mut self, buf: &mut impl Buf) -> DecodeResult<()>; + + fn parse_next_length_delimited( + &mut self, + wire_type: WireType, + buf: &mut impl Buf, + ) -> DecodeResult<()> { + check_wire_type(WireType::LengthDelimited, wire_type)?; + let len = decode_varint(buf)?; + let remaining = buf.remaining(); + if len > remaining as u64 { + return Err(DecodeError::InvalidData); + } + + let limit = remaining - len as usize; + while buf.remaining() > limit { + self.parse_next_field(buf)?; + } + + if buf.remaining() != limit { + return Err(DecodeError::InvalidData); + } + + Ok(()) + } +} diff --git a/betterproto-extras/src/decode/value.rs b/betterproto-extras/src/decode/value.rs new file mode 100644 index 000000000..6c3fb0028 --- /dev/null +++ b/betterproto-extras/src/decode/value.rs @@ -0,0 +1,293 @@ +use super::{ + custom_message::CustomMessageBuilder, map_entry::MapEntryBuilder, DecodeResult, MessageBuilder, +}; +use crate::{ + betterproto_interop::InteropResult, + descriptors::ProtoType, + well_known_types::{ + BoolValue, BytesValue, DoubleValue, Duration, FloatValue, Int32Value, Int64Value, + StringValue, Timestamp, UInt32Value, UInt64Value, + }, +}; +use prost::{ + bytes::Buf, + encoding::{self as enc, DecodeContext, WireType}, + Message, +}; +use pyo3::{ + types::{IntoPyDict, PyBytes}, + IntoPy, PyObject, Python, ToPyObject, +}; + +pub struct ValueBuilder<'a, 'py> { + py: Python<'py>, + proto_type: &'a ProtoType, + value: Value, +} +enum Value { + Unset, + Single(PyObject), + Repeated(Vec), + Map(Vec<(PyObject, PyObject)>), +} + +impl<'a, 'py> ValueBuilder<'a, 'py> { + pub fn new(py: Python<'py>, proto_type: &'a ProtoType) -> Self { + ValueBuilder { + py, + proto_type, + value: Value::Unset, + } + } + + pub fn reset(&mut self) { + self.value = Value::Unset; + } + + pub fn into_object(self) -> Option { + let py = self.py; + match self.value { + Value::Unset => None, + Value::Single(obj) => Some(obj), + Value::Repeated(ls) => Some(ls.to_object(py)), + Value::Map(ls) => Some(ls.into_py_dict(py).to_object(py)), + } + } + + pub fn into_object_or_default(self) -> DecodeResult { + let py = self.py; + let proto_type = self.proto_type; + self.into_object() + .map(Ok) + .unwrap_or_else(|| Ok(proto_type.default_value(py)?)) + } + + pub fn parse_next_single( + &mut self, + wire_type: WireType, + buf: &mut impl Buf, + ) -> DecodeResult<()> { + self.set_single(parse_next_value(self.py, self.proto_type, wire_type, buf)?); + Ok(()) + } + + pub fn parse_next_list_entry( + &mut self, + wire_type: WireType, + buf: &mut impl Buf, + ) -> DecodeResult<()> { + if let WireType::LengthDelimited = wire_type { + if let Some(obs) = try_parse_next_packed(self.py, self.proto_type, buf)? { + self.append_repeated(obs); + return Ok(()); + } + } + let obj = parse_next_value(self.py, self.proto_type, wire_type, buf)?; + self.push_repeated(obj); + Ok(()) + } + + pub fn parse_next_map_entry( + &mut self, + wire_type: WireType, + key_type: &ProtoType, + buf: &mut impl Buf, + ) -> DecodeResult<()> { + let mut builder = MapEntryBuilder::new(self.py, key_type, self.proto_type); + builder.parse_next_length_delimited(wire_type, buf)?; + self.push_map_entry(builder.into_tuple()?); + Ok(()) + } + + fn set_single(&mut self, obj: PyObject) { + match &mut self.value { + Value::Single(x) => *x = obj, + _ => self.value = Value::Single(obj), + } + } + + fn push_repeated(&mut self, obj: PyObject) { + match &mut self.value { + Value::Repeated(ls) => ls.push(obj), + _ => self.value = Value::Repeated(vec![obj]), + } + } + + fn append_repeated(&mut self, mut objs: Vec) { + match &mut self.value { + Value::Repeated(ls) => ls.append(&mut objs), + _ => self.value = Value::Repeated(objs), + } + } + + fn push_map_entry(&mut self, obj: (PyObject, PyObject)) { + match &mut self.value { + Value::Map(ls) => ls.push(obj), + _ => self.value = Value::Map(vec![obj]), + } + } +} + +fn parse_next_value( + py: Python, + proto_type: &ProtoType, + wire_type: WireType, + buf: &mut impl Buf, +) -> DecodeResult { + let ctx = Default::default(); + match proto_type { + ProtoType::Bool => { + let mut value = Default::default(); + enc::bool::merge(wire_type, &mut value, buf, ctx)?; + Ok(value.into_py(py)) + } + ProtoType::Bytes => { + let mut value = vec![]; + enc::bytes::merge(wire_type, &mut value, buf, ctx)?; + Ok(PyBytes::new(py, &value).to_object(py)) + } + ProtoType::Double => { + let mut value = Default::default(); + enc::double::merge(wire_type, &mut value, buf, ctx)?; + Ok(value.into_py(py)) + } + ProtoType::Float => { + let mut value = Default::default(); + enc::float::merge(wire_type, &mut value, buf, ctx)?; + Ok(value.into_py(py)) + } + ProtoType::Fixed32 => { + let mut value = Default::default(); + enc::fixed32::merge(wire_type, &mut value, buf, ctx)?; + Ok(value.into_py(py)) + } + ProtoType::Fixed64 => { + let mut value = Default::default(); + enc::fixed64::merge(wire_type, &mut value, buf, ctx)?; + Ok(value.into_py(py)) + } + ProtoType::Int32 => { + let mut value = Default::default(); + enc::int32::merge(wire_type, &mut value, buf, ctx)?; + Ok(value.into_py(py)) + } + ProtoType::Int64 => { + let mut value = Default::default(); + enc::int64::merge(wire_type, &mut value, buf, ctx)?; + Ok(value.into_py(py)) + } + ProtoType::Sfixed32 => { + let mut value = Default::default(); + enc::sfixed32::merge(wire_type, &mut value, buf, ctx)?; + Ok(value.into_py(py)) + } + ProtoType::Sfixed64 => { + let mut value = Default::default(); + enc::sfixed64::merge(wire_type, &mut value, buf, ctx)?; + Ok(value.into_py(py)) + } + ProtoType::Sint32 => { + let mut value = Default::default(); + enc::sint32::merge(wire_type, &mut value, buf, ctx)?; + Ok(value.into_py(py)) + } + ProtoType::Sint64 => { + let mut value = Default::default(); + enc::sint64::merge(wire_type, &mut value, buf, ctx)?; + Ok(value.into_py(py)) + } + ProtoType::String => { + let mut value = Default::default(); + enc::string::merge(wire_type, &mut value, buf, ctx)?; + Ok(value.into_py(py)) + } + ProtoType::Uint32 => { + let mut value = Default::default(); + enc::uint32::merge(wire_type, &mut value, buf, ctx)?; + Ok(value.into_py(py)) + } + ProtoType::Uint64 => { + let mut value = Default::default(); + enc::uint64::merge(wire_type, &mut value, buf, ctx)?; + Ok(value.into_py(py)) + } + ProtoType::Enum(cls) => { + let mut value = Default::default(); + enc::int32::merge(wire_type, &mut value, buf, ctx)?; + Ok(cls.create_instance(py, value)?) + } + ProtoType::CustomMessage(cls) => { + let mut builder = CustomMessageBuilder::new(py, cls.descriptor(py)?); + builder.parse_next_length_delimited(wire_type, buf)?; + let msg = cls.create_instance(py)?; + builder.merge_into(msg)?; + Ok(msg.to_object(py)) + } + ProtoType::BoolValue => Ok(BoolValue::decode_length_delimited(buf)?.to_object(py)), + ProtoType::BytesValue => Ok(BytesValue::decode_length_delimited(buf)?.to_object(py)), + ProtoType::DoubleValue => Ok(DoubleValue::decode_length_delimited(buf)?.to_object(py)), + ProtoType::FloatValue => Ok(FloatValue::decode_length_delimited(buf)?.to_object(py)), + ProtoType::Int32Value => Ok(Int32Value::decode_length_delimited(buf)?.to_object(py)), + ProtoType::Int64Value => Ok(Int64Value::decode_length_delimited(buf)?.to_object(py)), + ProtoType::UInt32Value => Ok(UInt32Value::decode_length_delimited(buf)?.to_object(py)), + ProtoType::UInt64Value => Ok(UInt64Value::decode_length_delimited(buf)?.to_object(py)), + ProtoType::StringValue => Ok(StringValue::decode_length_delimited(buf)?.to_object(py)), + ProtoType::Timestamp => Ok(Timestamp::decode_length_delimited(buf)?.to_object(py)), + ProtoType::Duration => Ok(Duration::decode_length_delimited(buf)?.to_object(py)), + } +} + +fn try_parse_next_packed( + py: Python, + proto_type: &ProtoType, + buf: &mut impl Buf, +) -> DecodeResult>> { + match proto_type { + ProtoType::Bool => Some(parse_next_packed(py, enc::bool::merge_repeated, buf)), + ProtoType::Double => Some(parse_next_packed(py, enc::double::merge_repeated, buf)), + ProtoType::Float => Some(parse_next_packed(py, enc::float::merge_repeated, buf)), + ProtoType::Fixed32 => Some(parse_next_packed(py, enc::fixed32::merge_repeated, buf)), + ProtoType::Fixed64 => Some(parse_next_packed(py, enc::fixed64::merge_repeated, buf)), + ProtoType::Sfixed32 => Some(parse_next_packed(py, enc::sfixed32::merge_repeated, buf)), + ProtoType::Sfixed64 => Some(parse_next_packed(py, enc::sfixed64::merge_repeated, buf)), + ProtoType::Int32 => Some(parse_next_packed(py, enc::int32::merge_repeated, buf)), + ProtoType::Int64 => Some(parse_next_packed(py, enc::int64::merge_repeated, buf)), + ProtoType::Uint32 => Some(parse_next_packed(py, enc::uint32::merge_repeated, buf)), + ProtoType::Uint64 => Some(parse_next_packed(py, enc::uint64::merge_repeated, buf)), + ProtoType::Sint32 => Some(parse_next_packed(py, enc::sint32::merge_repeated, buf)), + ProtoType::Sint64 => Some(parse_next_packed(py, enc::sint64::merge_repeated, buf)), + ProtoType::Enum(cls) => { + let mut ls = vec![]; + enc::int32::merge_repeated( + WireType::LengthDelimited, + &mut ls, + buf, + Default::default(), + )?; + let res = ls + .into_iter() + .map(|x| cls.create_instance(py, x)) + .collect::>>()?; + Some(Ok(res)) + } + _ => None, + } + .transpose() +} + +fn parse_next_packed(py: Python, merger: R, buf: &mut B) -> DecodeResult> +where + B: Buf, + R: FnOnce(WireType, &mut Vec, &mut B, DecodeContext) -> Result<(), prost::DecodeError>, + T: ToPyObject, +{ + let mut ls = vec![]; + merger( + WireType::LengthDelimited, + &mut ls, + buf, + DecodeContext::default(), + )?; + let res = ls.into_iter().map(|x| x.to_object(py)).collect::>(); + Ok(res) +} diff --git a/betterproto-extras/src/descriptors.rs b/betterproto-extras/src/descriptors.rs new file mode 100644 index 000000000..45245d004 --- /dev/null +++ b/betterproto-extras/src/descriptors.rs @@ -0,0 +1,97 @@ +use pyo3::{ + types::{PyBytes, PyString}, + PyObject, Python, ToPyObject, +}; + +use crate::{ + betterproto_interop::{BetterprotoEnumClass, BetterprotoMessageClass, InteropResult}, + well_known_types::{Duration, Timestamp}, + Str, +}; + +#[derive(Debug)] +pub struct MessageDescriptor { + pub fields: Vec<(u32, FieldDescriptor)>, +} + +#[derive(Debug)] +pub struct FieldDescriptor { + pub name: Str, + pub attribute: FieldAttribute, + pub value_type: ProtoType, +} + +#[derive(Debug)] +pub enum FieldAttribute { + None, + Optional, + Group(Str), + Map(ProtoType), + Repeated, +} + +#[derive(Debug)] +pub enum ProtoType { + Bool, + Bytes, + Int32, + Int64, + Uint32, + Uint64, + Float, + Double, + String, + Enum(BetterprotoEnumClass), + CustomMessage(BetterprotoMessageClass), + Sint32, + Sint64, + Fixed32, + Sfixed32, + Fixed64, + Sfixed64, + BoolValue, + BytesValue, + DoubleValue, + FloatValue, + Int32Value, + Int64Value, + UInt32Value, + UInt64Value, + StringValue, + Duration, + Timestamp, +} + +impl ProtoType { + pub fn default_value(&self, py: Python) -> InteropResult { + match self { + Self::Bool => Ok(false.to_object(py)), + Self::Bytes => Ok(PyBytes::new(py, &[]).to_object(py)), + Self::Double | Self::Float => Ok(0_f64.to_object(py)), + Self::Int32 + | Self::Int64 + | Self::Sint32 + | Self::Sint64 + | Self::Uint32 + | Self::Uint64 + | Self::Fixed32 + | Self::Fixed64 + | Self::Sfixed32 + | Self::Sfixed64 => Ok(0_i64.to_object(py)), + Self::String => Ok(PyString::new(py, "").to_object(py)), + Self::Enum(cls) => cls.create_instance(py, 0), + Self::CustomMessage(cls) => Ok(cls.create_instance(py)?.to_object(py)), + Self::BoolValue + | Self::BytesValue + | Self::FloatValue + | Self::DoubleValue + | Self::Int32Value + | Self::Int64Value + | Self::StringValue + | Self::UInt32Value + | Self::UInt64Value => Ok(py.None()), + Self::Timestamp => Ok(Timestamp::default().to_object(py)), + Self::Duration => Ok(Duration::default().to_object(py)), + } + } +} diff --git a/betterproto-extras/src/encode/chunk.rs b/betterproto-extras/src/encode/chunk.rs new file mode 100644 index 000000000..5bb877f81 --- /dev/null +++ b/betterproto-extras/src/encode/chunk.rs @@ -0,0 +1,64 @@ +use super::{message::MessageEncoder, EncodeResult}; +use prost::{bytes::BufMut, encoding as enc, Message}; + +pub struct Chunk(ChunkVariant); + +enum ChunkVariant { + PreEncoded(Box<[u8]>), + MessageField(u32, Box), +} + +impl Chunk { + pub fn from_encoder( + tag: u32, + value: &T, + len_fn: impl FnOnce(u32, &T) -> usize, + encoder: impl FnOnce(u32, &T, &mut Vec), + ) -> EncodeResult { + let capacity = len_fn(tag, value); + let mut buf = Vec::with_capacity(capacity); + encoder(tag, value, &mut buf); + debug_assert_eq!(capacity, buf.len()); + Ok(Self(ChunkVariant::PreEncoded(buf.into_boxed_slice()))) + } + + pub fn from_encoded(encoded: Vec) -> Self { + Self(ChunkVariant::PreEncoded(encoded.into_boxed_slice())) + } + + pub fn from_known_message(tag: u32, msg: T) -> EncodeResult { + let msg_len = msg.encoded_len(); + let capacity = msg_len + enc::key_len(tag) + enc::encoded_len_varint(msg_len as u64); + let mut buf = Vec::with_capacity(capacity); + enc::encode_key(tag, enc::WireType::LengthDelimited, &mut buf); + msg.encode_length_delimited(&mut buf)?; + debug_assert_eq!(capacity, buf.len()); + Ok(Self(ChunkVariant::PreEncoded(buf.into_boxed_slice()))) + } + + pub fn from_message(tag: u32, encoder: MessageEncoder) -> Self { + Self(ChunkVariant::MessageField(tag, Box::new(encoder))) + } + + pub fn encoded_len(&self) -> usize { + match &self.0 { + ChunkVariant::PreEncoded(bytes) => bytes.len(), + ChunkVariant::MessageField(tag, msg) => { + let msg_len = msg.encoded_len(); + let meta_size = enc::key_len(*tag) + enc::encoded_len_varint(msg_len as u64); + msg_len + meta_size + } + } + } + + pub fn encode(&self, buf: &mut Vec) { + match &self.0 { + ChunkVariant::PreEncoded(bytes) => buf.put_slice(bytes), + ChunkVariant::MessageField(tag, msg) => { + enc::encode_key(*tag, enc::WireType::LengthDelimited, buf); + enc::encode_varint(msg.encoded_len() as u64, buf); + msg.encode(buf); + } + } + } +} diff --git a/betterproto-extras/src/encode/error.rs b/betterproto-extras/src/encode/error.rs new file mode 100644 index 000000000..2380cf0e7 --- /dev/null +++ b/betterproto-extras/src/encode/error.rs @@ -0,0 +1,29 @@ +use crate::betterproto_interop::InteropError; +use pyo3::{exceptions::PyRuntimeError, PyDowncastError, PyErr}; +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum EncodeError { + #[error("Given object is not a valid betterproto message.")] + NoBetterprotoMessage(#[from] PyErr), + #[error("Given object is not a valid betterproto message.")] + DowncastFailed, + #[error(transparent)] + Interop(#[from] InteropError), + #[error("Given object is not a valid betterproto message.")] + ProstEncode(#[from] prost::EncodeError), +} + +pub type EncodeResult = Result; + +impl From> for EncodeError { + fn from(_: PyDowncastError) -> Self { + Self::DowncastFailed + } +} + +impl From for PyErr { + fn from(value: EncodeError) -> Self { + PyRuntimeError::new_err(value.to_string()) + } +} diff --git a/betterproto-extras/src/encode/message.rs b/betterproto-extras/src/encode/message.rs new file mode 100644 index 000000000..8a6a5347b --- /dev/null +++ b/betterproto-extras/src/encode/message.rs @@ -0,0 +1,403 @@ +use super::{chunk::Chunk, EncodeResult}; +use crate::{ + betterproto_interop::BetterprotoMessage, + descriptors::{FieldAttribute, FieldDescriptor, MessageDescriptor, ProtoType}, + well_known_types::{ + BoolValue, BytesValue, DoubleValue, Duration, FloatValue, Int32Value, Int64Value, + StringValue, Timestamp, UInt32Value, UInt64Value, + }, +}; +use prost::{encoding as enc, Message}; +use pyo3::{ + intern, + types::{PyDict, PyList}, + PyAny, PyResult, +}; + +pub struct MessageEncoder(Vec); + +impl MessageEncoder { + pub fn from_betterproto_msg( + msg: BetterprotoMessage, + descriptor: &MessageDescriptor, + ) -> EncodeResult { + let mut encoder = MessageEncoder::new(); + for (tag, field) in descriptor.fields.iter() { + if let Some(value) = msg.get_field(&field.name)? { + encoder.load_field(*tag, field, value)?; + } + } + encoder.load_unknown_fields(msg.get_unknown_fields()?); + Ok(encoder) + } + + pub fn into_vec(self) -> Vec { + let capacity = self.encoded_len(); + let mut buf = Vec::with_capacity(capacity); + self.encode(&mut buf); + debug_assert_eq!(capacity, buf.len()); + buf + } + + fn new() -> Self { + Self(vec![]) + } + + pub(super) fn encoded_len(&self) -> usize { + self.0 + .iter() + .map(|chunk| chunk.encoded_len()) + .sum::() + } + + pub(super) fn encode(&self, buf: &mut Vec) { + for chunk in self.0.iter() { + chunk.encode(buf); + } + } + + fn load_unknown_fields(&mut self, unknowns: Vec) { + self.0.push(Chunk::from_encoded(unknowns)) + } + + fn load_field( + &mut self, + tag: u32, + descriptor: &FieldDescriptor, + value: &PyAny, + ) -> EncodeResult<()> { + match &descriptor.attribute { + FieldAttribute::Repeated => { + if !self.try_load_packed(tag, &descriptor.value_type, value)? { + for value in value.downcast::()?.iter() { + self.load_single::(tag, &descriptor.value_type, value)?; + } + } + } + FieldAttribute::Map(key_type) => { + for (key, value) in value.downcast::()?.iter() { + self.load_map_entry(tag, key_type, &descriptor.value_type, key, value)?; + } + } + FieldAttribute::None => self.load_single::(tag, &descriptor.value_type, value)?, + _ => self.load_single::(tag, &descriptor.value_type, value)?, + } + + Ok(()) + } + + fn load_single( + &mut self, + tag: u32, + proto_type: &ProtoType, + value: &PyAny, + ) -> EncodeResult<()> { + let py = value.py(); + let chunk = match proto_type { + ProtoType::Bool => { + let value: bool = value.extract()?; + if SKIP_DEFAULT && !value { + return Ok(()); + } + Chunk::from_encoder(tag, &value, enc::bool::encoded_len, enc::bool::encode)? + } + ProtoType::Bytes => { + let value: Vec = value.extract()?; + if SKIP_DEFAULT && value.is_empty() { + return Ok(()); + } + Chunk::from_encoder(tag, &value, enc::bytes::encoded_len, enc::bytes::encode)? + } + ProtoType::Double => { + let value: f64 = value.extract()?; + if SKIP_DEFAULT && value == 0.0 { + return Ok(()); + } + Chunk::from_encoder(tag, &value, enc::double::encoded_len, enc::double::encode)? + } + ProtoType::Float => { + let value: f32 = value.extract()?; + if SKIP_DEFAULT && value == 0.0 { + return Ok(()); + } + Chunk::from_encoder(tag, &value, enc::float::encoded_len, enc::float::encode)? + } + ProtoType::Fixed32 => { + let value: u32 = value.extract()?; + if SKIP_DEFAULT && value == 0 { + return Ok(()); + } + Chunk::from_encoder(tag, &value, enc::fixed32::encoded_len, enc::fixed32::encode)? + } + ProtoType::Fixed64 => { + let value: u64 = value.extract()?; + if SKIP_DEFAULT && value == 0 { + return Ok(()); + } + Chunk::from_encoder(tag, &value, enc::fixed64::encoded_len, enc::fixed64::encode)? + } + ProtoType::Int32 => { + let value: i32 = value.extract()?; + if SKIP_DEFAULT && value == 0 { + return Ok(()); + } + Chunk::from_encoder(tag, &value, enc::int32::encoded_len, enc::int32::encode)? + } + ProtoType::Int64 => { + let value: i64 = value.extract()?; + if SKIP_DEFAULT && value == 0 { + return Ok(()); + } + Chunk::from_encoder(tag, &value, enc::int64::encoded_len, enc::int64::encode)? + } + ProtoType::Sfixed32 => { + let value: i32 = value.extract()?; + if SKIP_DEFAULT && value == 0 { + return Ok(()); + } + Chunk::from_encoder( + tag, + &value, + enc::sfixed32::encoded_len, + enc::sfixed32::encode, + )? + } + ProtoType::Sfixed64 => { + let value: i64 = value.extract()?; + if SKIP_DEFAULT && value == 0 { + return Ok(()); + } + Chunk::from_encoder( + tag, + &value, + enc::sfixed64::encoded_len, + enc::sfixed64::encode, + )? + } + ProtoType::Sint32 => { + let value: i32 = value.extract()?; + if SKIP_DEFAULT && value == 0 { + return Ok(()); + } + Chunk::from_encoder(tag, &value, enc::sint32::encoded_len, enc::sint32::encode)? + } + ProtoType::Sint64 => { + let value: i64 = value.extract()?; + if SKIP_DEFAULT && value == 0 { + return Ok(()); + } + Chunk::from_encoder(tag, &value, enc::sint64::encoded_len, enc::sint64::encode)? + } + ProtoType::String => { + let value: String = value.extract()?; + if SKIP_DEFAULT && value.is_empty() { + return Ok(()); + } + Chunk::from_encoder(tag, &value, enc::string::encoded_len, enc::string::encode)? + } + ProtoType::Uint32 => { + let value: u32 = value.extract()?; + if SKIP_DEFAULT && value == 0 { + return Ok(()); + } + Chunk::from_encoder(tag, &value, enc::uint32::encoded_len, enc::uint32::encode)? + } + ProtoType::Uint64 => { + let value: u64 = value.extract()?; + if SKIP_DEFAULT && value == 0 { + return Ok(()); + } + Chunk::from_encoder(tag, &value, enc::uint64::encoded_len, enc::uint64::encode)? + } + ProtoType::Enum(_) => { + let value: i32 = value + .getattr(intern!(py, "value")) + .unwrap_or(value) + .extract()?; + if SKIP_DEFAULT && value == 0 { + return Ok(()); + } + Chunk::from_encoder(tag, &value, enc::int32::encoded_len, enc::int32::encode)? + } + ProtoType::CustomMessage(cls) => { + let msg: BetterprotoMessage = value.extract()?; + if SKIP_DEFAULT && !msg.should_be_serialized()? { + return Ok(()); + } + Chunk::from_message( + tag, + MessageEncoder::from_betterproto_msg(msg, cls.descriptor(py)?)?, + ) + } + ProtoType::BoolValue => Chunk::from_known_message::(tag, value.extract()?)?, + ProtoType::BytesValue => { + Chunk::from_known_message::(tag, value.extract()?)? + } + ProtoType::DoubleValue => { + Chunk::from_known_message::(tag, value.extract()?)? + } + ProtoType::FloatValue => { + Chunk::from_known_message::(tag, value.extract()?)? + } + ProtoType::Int32Value => { + Chunk::from_known_message::(tag, value.extract()?)? + } + ProtoType::Int64Value => { + Chunk::from_known_message::(tag, value.extract()?)? + } + ProtoType::UInt32Value => { + Chunk::from_known_message::(tag, value.extract()?)? + } + ProtoType::UInt64Value => { + Chunk::from_known_message::(tag, value.extract()?)? + } + ProtoType::StringValue => { + Chunk::from_known_message::(tag, value.extract()?)? + } + ProtoType::Timestamp => { + let msg: Timestamp = value.extract()?; + if SKIP_DEFAULT && msg.encoded_len() == 0 { + return Ok(()); + } + Chunk::from_known_message(tag, msg)? + } + ProtoType::Duration => { + let msg: Duration = value.extract()?; + if SKIP_DEFAULT && msg.encoded_len() == 0 { + return Ok(()); + } + Chunk::from_known_message(tag, msg)? + } + }; + + self.0.push(chunk); + Ok(()) + } + + fn try_load_packed( + &mut self, + tag: u32, + proto_type: &ProtoType, + value: &PyAny, + ) -> EncodeResult { + let chunk = match proto_type { + ProtoType::Bool => Some(Chunk::from_encoder( + tag, + value.extract::>()?.as_ref(), + enc::bool::encoded_len_packed, + enc::bool::encode_packed, + )?), + ProtoType::Double => Some(Chunk::from_encoder( + tag, + value.extract::>()?.as_ref(), + enc::double::encoded_len_packed, + enc::double::encode_packed, + )?), + ProtoType::Float => Some(Chunk::from_encoder( + tag, + value.extract::>()?.as_ref(), + enc::float::encoded_len_packed, + enc::float::encode_packed, + )?), + ProtoType::Fixed32 => Some(Chunk::from_encoder( + tag, + value.extract::>()?.as_ref(), + enc::fixed32::encoded_len_packed, + enc::fixed32::encode_packed, + )?), + ProtoType::Fixed64 => Some(Chunk::from_encoder( + tag, + value.extract::>()?.as_ref(), + enc::fixed64::encoded_len_packed, + enc::fixed64::encode_packed, + )?), + ProtoType::Sfixed32 => Some(Chunk::from_encoder( + tag, + value.extract::>()?.as_ref(), + enc::sfixed32::encoded_len_packed, + enc::sfixed32::encode_packed, + )?), + ProtoType::Sfixed64 => Some(Chunk::from_encoder( + tag, + value.extract::>()?.as_ref(), + enc::sfixed64::encoded_len_packed, + enc::sfixed64::encode_packed, + )?), + ProtoType::Int32 => Some(Chunk::from_encoder( + tag, + value.extract::>()?.as_ref(), + enc::int32::encoded_len_packed, + enc::int32::encode_packed, + )?), + ProtoType::Int64 => Some(Chunk::from_encoder( + tag, + value.extract::>()?.as_ref(), + enc::int64::encoded_len_packed, + enc::int64::encode_packed, + )?), + ProtoType::Uint32 => Some(Chunk::from_encoder( + tag, + value.extract::>()?.as_ref(), + enc::uint32::encoded_len_packed, + enc::uint32::encode_packed, + )?), + ProtoType::Uint64 => Some(Chunk::from_encoder( + tag, + value.extract::>()?.as_ref(), + enc::uint64::encoded_len_packed, + enc::uint64::encode_packed, + )?), + ProtoType::Sint32 => Some(Chunk::from_encoder( + tag, + value.extract::>()?.as_ref(), + enc::sint32::encoded_len_packed, + enc::sint32::encode_packed, + )?), + ProtoType::Sint64 => Some(Chunk::from_encoder( + tag, + value.extract::>()?.as_ref(), + enc::sint64::encoded_len_packed, + enc::sint64::encode_packed, + )?), + ProtoType::Enum(_) => Some(Chunk::from_encoder( + tag, + value + .downcast::()? + .iter() + .map(|x| { + x.getattr(intern!(x.py(), "value")) + .unwrap_or(x) + .extract::() + }) + .collect::>>()? + .as_ref(), + enc::int32::encoded_len_packed, + enc::int32::encode_packed, + )?), + _ => None, + }; + + match chunk { + Some(chunk) => { + self.0.push(chunk); + Ok(true) + } + _ => Ok(false), + } + } + + fn load_map_entry( + &mut self, + tag: u32, + key_type: &ProtoType, + value_type: &ProtoType, + key: &PyAny, + value: &PyAny, + ) -> EncodeResult<()> { + let mut encoder = MessageEncoder::new(); + encoder.load_single::(1, key_type, key)?; + encoder.load_single::(2, value_type, value)?; + self.0.push(Chunk::from_message(tag, encoder)); + Ok(()) + } +} diff --git a/betterproto-extras/src/encode/mod.rs b/betterproto-extras/src/encode/mod.rs new file mode 100644 index 000000000..c1e747ea9 --- /dev/null +++ b/betterproto-extras/src/encode/mod.rs @@ -0,0 +1,8 @@ +mod chunk; +mod error; +mod message; + +pub use self::{ + error::{EncodeError, EncodeResult}, + message::MessageEncoder, +}; diff --git a/betterproto-extras/src/lib.rs b/betterproto-extras/src/lib.rs new file mode 100644 index 000000000..86a1490d1 --- /dev/null +++ b/betterproto-extras/src/lib.rs @@ -0,0 +1,32 @@ +mod betterproto_interop; +mod decode; +mod descriptors; +mod encode; +mod well_known_types; + +use betterproto_interop::BetterprotoMessage; +use decode::{merge_into_message, DecodeResult}; +use encode::{EncodeResult, MessageEncoder}; +use pyo3::{prelude::*, types::PyBytes}; +use std::sync::Arc; + +pub type Str = Arc; + +#[pyfunction] +fn deserialize(obj: BetterprotoMessage, mut buf: &[u8]) -> DecodeResult<()> { + merge_into_message(obj, &mut buf) +} + +#[pyfunction] +fn serialize<'py>(py: Python<'py>, msg: BetterprotoMessage) -> EncodeResult<&'py PyBytes> { + let cls = msg.class(); + let encoder = MessageEncoder::from_betterproto_msg(msg, cls.descriptor(py)?)?; + Ok(PyBytes::new(py, &encoder.into_vec())) +} + +#[pymodule] +fn betterproto_extras(_py: Python, m: &PyModule) -> PyResult<()> { + m.add_function(wrap_pyfunction!(deserialize, m)?)?; + m.add_function(wrap_pyfunction!(serialize, m)?)?; + Ok(()) +} diff --git a/betterproto-extras/src/well_known_types.rs b/betterproto-extras/src/well_known_types.rs new file mode 100644 index 000000000..ec28507ca --- /dev/null +++ b/betterproto-extras/src/well_known_types.rs @@ -0,0 +1,326 @@ +use indoc::indoc; +use prost::Message; +use pyo3::{ + sync::GILOnceCell, + types::{PyBytes, PyModule}, + FromPyObject, PyAny, PyObject, PyResult, Python, ToPyObject, +}; + +#[derive(Message)] +pub struct BoolValue { + #[prost(bool, tag = "1")] + pub value: bool, +} + +#[derive(Message)] +pub struct BytesValue { + #[prost(bytes, tag = "1")] + pub value: Vec, +} + +#[derive(Message)] +pub struct DoubleValue { + #[prost(double, tag = "1")] + pub value: f64, +} + +#[derive(Message)] +pub struct FloatValue { + #[prost(float, tag = "1")] + pub value: f32, +} + +#[derive(Message)] +pub struct Int32Value { + #[prost(int32, tag = "1")] + pub value: i32, +} + +#[derive(Message)] +pub struct Int64Value { + #[prost(int64, tag = "1")] + pub value: i64, +} + +#[derive(Message)] +pub struct UInt32Value { + #[prost(uint32, tag = "1")] + pub value: u32, +} + +#[derive(Message)] +pub struct UInt64Value { + #[prost(uint64, tag = "1")] + pub value: u64, +} + +#[derive(Message)] +pub struct StringValue { + #[prost(string, tag = "1")] + pub value: String, +} + +#[derive(Message)] +pub struct Duration { + #[prost(int64, tag = "1")] + pub seconds: i64, + #[prost(int32, tag = "2")] + pub nanos: i32, +} + +#[derive(Message)] +pub struct Timestamp { + #[prost(int64, tag = "1")] + pub seconds: i64, + #[prost(int32, tag = "2")] + pub nanos: i32, +} + +impl<'py> FromPyObject<'py> for BoolValue { + fn extract(ob: &'py PyAny) -> PyResult { + let res = BoolValue { + value: ob.extract()?, + }; + Ok(res) + } +} + +impl<'py> FromPyObject<'py> for BytesValue { + fn extract(ob: &'py PyAny) -> PyResult { + let res = BytesValue { + value: ob.extract()?, + }; + Ok(res) + } +} + +impl<'py> FromPyObject<'py> for DoubleValue { + fn extract(ob: &'py PyAny) -> PyResult { + let res = DoubleValue { + value: ob.extract()?, + }; + Ok(res) + } +} + +impl<'py> FromPyObject<'py> for FloatValue { + fn extract(ob: &'py PyAny) -> PyResult { + let res = FloatValue { + value: ob.extract()?, + }; + Ok(res) + } +} + +impl<'py> FromPyObject<'py> for Int32Value { + fn extract(ob: &'py PyAny) -> PyResult { + let res = Int32Value { + value: ob.extract()?, + }; + Ok(res) + } +} + +impl<'py> FromPyObject<'py> for Int64Value { + fn extract(ob: &'py PyAny) -> PyResult { + let res = Int64Value { + value: ob.extract()?, + }; + Ok(res) + } +} + +impl<'py> FromPyObject<'py> for UInt32Value { + fn extract(ob: &'py PyAny) -> PyResult { + let res = UInt32Value { + value: ob.extract()?, + }; + Ok(res) + } +} + +impl<'py> FromPyObject<'py> for UInt64Value { + fn extract(ob: &'py PyAny) -> PyResult { + let res = UInt64Value { + value: ob.extract()?, + }; + Ok(res) + } +} + +impl<'py> FromPyObject<'py> for StringValue { + fn extract(ob: &'py PyAny) -> PyResult { + let res = StringValue { + value: ob.extract()?, + }; + Ok(res) + } +} + +impl<'py> FromPyObject<'py> for Duration { + fn extract(ob: &'py PyAny) -> PyResult { + let py = ob.py(); + static DECONSTRUCTOR_CACHE: GILOnceCell = GILOnceCell::new(); + let deconstructor = DECONSTRUCTOR_CACHE + .get_or_init(py, || { + PyModule::from_code( + py, + indoc! {" + from datetime import timedelta + + def deconstructor(delta, *, _1_microsecond = timedelta(microseconds=1)): + total_ms = delta // _1_microsecond + seconds = int(total_ms / 1e6) + nanos = int((total_ms % 1e6) * 1e3) + return (seconds, nanos) + "}, + "", + "", + ) + .expect("This is a valid Python module") + .getattr("deconstructor") + .expect("Attribute exists") + .to_object(py) + }) + .as_ref(py); + let (seconds, nanos) = deconstructor.call1((ob,))?.extract()?; + let res = Duration { seconds, nanos }; + Ok(res) + } +} + +impl<'py> FromPyObject<'py> for Timestamp { + fn extract(ob: &'py PyAny) -> PyResult { + let py = ob.py(); + static DECONSTRUCTOR_CACHE: GILOnceCell = GILOnceCell::new(); + let deconstructor = DECONSTRUCTOR_CACHE + .get_or_init(py, || { + PyModule::from_code( + py, + indoc! {" + def deconstructor(dt): + seconds = int(dt.timestamp()) + nanos = int(dt.microsecond * 1e3) + return (seconds, nanos) + "}, + "", + "", + ) + .expect("This is a valid Python module") + .getattr("deconstructor") + .expect("Attribute exists") + .to_object(py) + }) + .as_ref(py); + let (seconds, nanos) = deconstructor.call1((ob,))?.extract()?; + let res = Timestamp { seconds, nanos }; + Ok(res) + } +} + +impl ToPyObject for BoolValue { + fn to_object(&self, py: Python) -> PyObject { + self.value.to_object(py) + } +} + +impl ToPyObject for BytesValue { + fn to_object(&self, py: Python) -> PyObject { + PyBytes::new(py, &self.value).to_object(py) + } +} + +impl ToPyObject for DoubleValue { + fn to_object(&self, py: Python) -> PyObject { + self.value.to_object(py) + } +} + +impl ToPyObject for FloatValue { + fn to_object(&self, py: Python) -> PyObject { + self.value.to_object(py) + } +} + +impl ToPyObject for Int32Value { + fn to_object(&self, py: Python) -> PyObject { + self.value.to_object(py) + } +} + +impl ToPyObject for Int64Value { + fn to_object(&self, py: Python) -> PyObject { + self.value.to_object(py) + } +} + +impl ToPyObject for UInt32Value { + fn to_object(&self, py: Python) -> PyObject { + self.value.to_object(py) + } +} + +impl ToPyObject for UInt64Value { + fn to_object(&self, py: Python) -> PyObject { + self.value.to_object(py) + } +} + +impl ToPyObject for StringValue { + fn to_object(&self, py: Python) -> PyObject { + self.value.to_object(py) + } +} + +impl ToPyObject for Duration { + fn to_object(&self, py: Python) -> PyObject { + static CONSTRUCTOR_CACHE: GILOnceCell = GILOnceCell::new(); + let constructor = CONSTRUCTOR_CACHE.get_or_init(py, || { + PyModule::from_code( + py, + indoc! {" + from datetime import timedelta + + def constructor(s, ms): + return timedelta(seconds=s, microseconds=ms) + "}, + "", + "", + ) + .expect("This is a valid Python module") + .getattr("constructor") + .expect("Attribute exists") + .to_object(py) + }); + constructor + .call1(py, (self.seconds as f64, (self.nanos as f64) / 1e3)) + .expect("static function will not fail") + } +} + +impl ToPyObject for Timestamp { + fn to_object(&self, py: Python) -> PyObject { + static CONSTRUCTOR_CACHE: GILOnceCell = GILOnceCell::new(); + let constructor = CONSTRUCTOR_CACHE.get_or_init(py, || { + PyModule::from_code( + py, + indoc! {" + from datetime import datetime, timezone + + def constructor(ts): + return datetime.fromtimestamp(ts, tz=timezone.utc) + "}, + "", + "", + ) + .expect("This is a valid Python module") + .getattr("constructor") + .expect("Attribute exists") + .to_object(py) + }); + let ts = (self.seconds as f64) + (self.nanos as f64) / 1e9; + constructor + .call1(py, (ts,)) + .expect("static function will not fail") + } +} diff --git a/example.py b/example.py new file mode 100644 index 000000000..c129d951d --- /dev/null +++ b/example.py @@ -0,0 +1,55 @@ +# dev tests +# to be deleted later + +import betterproto +from dataclasses import dataclass +from typing import Dict, List, Optional + +@dataclass(repr=False) +class Baz(betterproto.Message): + a: float = betterproto.float_field(1, group = "x") + b: int = betterproto.int64_field(2, group = "x") + c: float = betterproto.float_field(3, group = "y") + d: int = betterproto.int64_field(4, group = "y") + e: Optional[int] = betterproto.int32_field(5, group = "_e", optional = True) + +@dataclass(repr=False) +class Foo(betterproto.Message): + x: int = betterproto.int32_field(1) + y: float = betterproto.double_field(2) + z: List[Baz] = betterproto.message_field(3) + +class Enm(betterproto.Enum): + A = 0 + B = 1 + C = 2 + +@dataclass(repr=False) +class Bar(betterproto.Message): + foo1: Foo = betterproto.message_field(1) + foo2: Foo = betterproto.message_field(2) + packed: List[int] = betterproto.int64_field(3) + enm: Enm = betterproto.enum_field(4) + map: Dict[int, bool] = betterproto.map_field(5, betterproto.TYPE_INT64, betterproto.TYPE_BOOL) + maybe: Optional[bool] = betterproto.message_field(6, wraps=betterproto.TYPE_BOOL) + bts: bytes = betterproto.bytes_field(7) + +# Native serialization happening here +buffer = bytes( + Bar( + foo1=Foo(1, 2.34), + foo2=Foo(3, 4.56, [Baz(a = 1.234), Baz(b = 5, e=1), Baz(b = 2, d = 3)]), + packed=[5, 3, 1], + enm=Enm.B, + map={ + 1: True, + 42: False + }, + maybe=True, + bts=b'Hi There!' + ) +) + +# Native deserialization happening here +bar = Bar().parse(buffer) +print(bar) diff --git a/poetry.lock b/poetry.lock index 00d196612..6dcd0330a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,10 +1,9 @@ -# This file is automatically @generated by Poetry and should not be changed by hand. +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. [[package]] name = "alabaster" version = "0.7.13" description = "A configurable sidebar-enabled Sphinx theme" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -16,7 +15,6 @@ files = [ name = "ansicon" version = "1.89.0" description = "Python wrapper for loading Jason Hood's ANSICON" -category = "dev" optional = false python-versions = "*" files = [ @@ -28,7 +26,6 @@ files = [ name = "asv" version = "0.4.2" description = "Airspeed Velocity: A simple Python history benchmarking tool" -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -45,7 +42,6 @@ hg = ["python-hglib (>=1.5)"] name = "atomicwrites" version = "1.4.1" description = "Atomic file writes." -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -56,7 +52,6 @@ files = [ name = "attrs" version = "23.1.0" description = "Classes Without Boilerplate" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -78,7 +73,6 @@ tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pyte name = "babel" version = "2.12.1" description = "Internationalization utilities" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -93,7 +87,6 @@ pytz = {version = ">=2015.7", markers = "python_version < \"3.9\""} name = "backports-cached-property" version = "1.0.2" description = "cached_property() - computed once per instance, cached as attribute" -category = "dev" optional = false python-versions = ">=3.6.0" files = [ @@ -101,11 +94,23 @@ files = [ {file = "backports.cached_property-1.0.2-py3-none-any.whl", hash = "sha256:baeb28e1cd619a3c9ab8941431fe34e8490861fb998c6c4590693d50171db0cc"}, ] +[[package]] +name = "betterproto-extras" +version = "0.1.0" +description = "" +optional = false +python-versions = ">=3.7" +files = [] +develop = false + +[package.source] +type = "directory" +url = "betterproto-extras" + [[package]] name = "black" version = "23.3.0" description = "The uncompromising code formatter." -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -156,7 +161,6 @@ uvloop = ["uvloop (>=0.15.2)"] name = "blessed" version = "1.20.0" description = "Easy, practical library for making terminal apps, by providing an elegant, well-documented interface to Colors, Keyboard input, and screen Positioning capabilities." -category = "dev" optional = false python-versions = ">=2.7" files = [ @@ -173,7 +177,6 @@ wcwidth = ">=0.1.4" name = "bpython" version = "0.19" description = "Fancy Interface to the Python Interpreter" -category = "dev" optional = false python-versions = "*" files = [ @@ -197,7 +200,6 @@ watch = ["watchdog"] name = "certifi" version = "2023.5.7" description = "Python package for providing Mozilla's CA Bundle." -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -209,7 +211,6 @@ files = [ name = "cfgv" version = "3.3.1" description = "Validate configuration and produce human readable error messages." -category = "dev" optional = false python-versions = ">=3.6.1" files = [ @@ -221,7 +222,6 @@ files = [ name = "charset-normalizer" version = "3.1.0" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -category = "dev" optional = false python-versions = ">=3.7.0" files = [ @@ -306,7 +306,6 @@ files = [ name = "click" version = "8.1.3" description = "Composable command line interface toolkit" -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -322,7 +321,6 @@ importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" files = [ @@ -334,7 +332,6 @@ files = [ name = "coverage" version = "7.2.7" description = "Code coverage measurement for Python" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -407,7 +404,6 @@ toml = ["tomli"] name = "curtsies" version = "0.4.1" description = "Curses-like terminal wrapper, with colored strings!" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -423,7 +419,6 @@ cwcwidth = "*" name = "cwcwidth" version = "0.1.8" description = "Python bindings for wc(s)width" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -469,7 +464,6 @@ files = [ name = "distlib" version = "0.3.6" description = "Distribution utilities" -category = "dev" optional = false python-versions = "*" files = [ @@ -481,7 +475,6 @@ files = [ name = "docutils" version = "0.20.1" description = "Docutils -- Python Documentation Utilities" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -493,7 +486,6 @@ files = [ name = "filelock" version = "3.12.0" description = "A platform independent file lock." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -509,7 +501,6 @@ testing = ["covdefaults (>=2.3)", "coverage (>=7.2.3)", "diff-cover (>=7.5)", "p name = "greenlet" version = "2.0.2" description = "Lightweight in-process concurrent programming" -category = "dev" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*" files = [ @@ -583,7 +574,6 @@ test = ["objgraph", "psutil"] name = "grpcio" version = "1.54.2" description = "HTTP/2-based RPC framework" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -641,7 +631,6 @@ protobuf = ["grpcio-tools (>=1.54.2)"] name = "grpcio-tools" version = "1.54.2" description = "Protobuf code generator for gRPC" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -701,7 +690,6 @@ setuptools = "*" name = "grpclib" version = "0.4.4" description = "Pure-Python gRPC implementation for asyncio" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -719,7 +707,6 @@ protobuf = ["protobuf (>=3.15.0)"] name = "h2" version = "4.1.0" description = "HTTP/2 State-Machine based protocol implementation" -category = "main" optional = false python-versions = ">=3.6.1" files = [ @@ -735,7 +722,6 @@ hyperframe = ">=6.0,<7" name = "hpack" version = "4.0.0" description = "Pure-Python HPACK header compression" -category = "main" optional = false python-versions = ">=3.6.1" files = [ @@ -747,7 +733,6 @@ files = [ name = "hyperframe" version = "6.0.1" description = "HTTP/2 framing layer for Python" -category = "main" optional = false python-versions = ">=3.6.1" files = [ @@ -759,7 +744,6 @@ files = [ name = "identify" version = "2.5.24" description = "File identification library for Python" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -774,7 +758,6 @@ license = ["ukkonen"] name = "idna" version = "3.4" description = "Internationalized Domain Names in Applications (IDNA)" -category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -786,7 +769,6 @@ files = [ name = "imagesize" version = "1.4.1" description = "Getting image size from png/jpeg/jpeg2000/gif file" -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -798,7 +780,6 @@ files = [ name = "importlib-metadata" version = "6.6.0" description = "Read metadata from Python packages" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -819,7 +800,6 @@ testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packag name = "iniconfig" version = "2.0.0" description = "brain-dead simple config-ini parsing" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -831,7 +811,6 @@ files = [ name = "isort" version = "5.11.5" description = "A Python utility / library to sort Python imports." -category = "main" optional = true python-versions = ">=3.7.0" files = [ @@ -849,7 +828,6 @@ requirements-deprecated-finder = ["pip-api", "pipreqs"] name = "jinja2" version = "3.1.2" description = "A very fast and expressive template engine." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -867,7 +845,6 @@ i18n = ["Babel (>=2.7)"] name = "jinxed" version = "1.2.0" description = "Jinxed Terminal Library" -category = "dev" optional = false python-versions = "*" files = [ @@ -882,7 +859,6 @@ ansicon = {version = "*", markers = "platform_system == \"Windows\""} name = "markupsafe" version = "2.1.2" description = "Safely add untrusted strings to HTML/XML markup." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -942,7 +918,6 @@ files = [ name = "multidict" version = "6.0.4" description = "multidict implementation" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1026,7 +1001,6 @@ files = [ name = "mypy" version = "0.930" description = "Optional static typing for Python" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1066,7 +1040,6 @@ python2 = ["typed-ast (>=1.4.0,<2)"] name = "mypy-extensions" version = "1.0.0" description = "Type system extensions for programs checked with the mypy type checker." -category = "main" optional = false python-versions = ">=3.5" files = [ @@ -1078,7 +1051,6 @@ files = [ name = "nodeenv" version = "1.8.0" description = "Node.js virtual environment builder" -category = "dev" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" files = [ @@ -1093,7 +1065,6 @@ setuptools = "*" name = "packaging" version = "23.1" description = "Core utilities for Python packages" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1105,7 +1076,6 @@ files = [ name = "pastel" version = "0.2.1" description = "Bring colors to your terminal." -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -1117,7 +1087,6 @@ files = [ name = "pathspec" version = "0.11.1" description = "Utility library for gitignore style pattern matching of file paths." -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -1129,7 +1098,6 @@ files = [ name = "platformdirs" version = "3.5.1" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1148,7 +1116,6 @@ test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.3.1)", "pytest- name = "pluggy" version = "1.0.0" description = "plugin and hook calling mechanisms for python" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1167,7 +1134,6 @@ testing = ["pytest", "pytest-benchmark"] name = "poethepoet" version = "0.19.0" description = "A task runner that works well with poetry." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1186,7 +1152,6 @@ poetry-plugin = ["poetry (>=1.0,<2.0)"] name = "pre-commit" version = "2.21.0" description = "A framework for managing and maintaining multi-language pre-commit hooks." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1206,7 +1171,6 @@ virtualenv = ">=20.10.0" name = "protobuf" version = "4.23.2" description = "" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1229,7 +1193,6 @@ files = [ name = "py" version = "1.11.0" description = "library with cross-python path, ini-parsing, io, code, log facilities" -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -1241,7 +1204,6 @@ files = [ name = "pydantic" version = "1.10.8" description = "Data validation and settings management using python type hints" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1294,7 +1256,6 @@ email = ["email-validator (>=1.0.3)"] name = "pygments" version = "2.15.1" description = "Pygments is a syntax highlighting package written in Python." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1309,7 +1270,6 @@ plugins = ["importlib-metadata"] name = "pytest" version = "6.2.5" description = "pytest: simple powerful testing with Python" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1335,7 +1295,6 @@ testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xm name = "pytest-asyncio" version = "0.12.0" description = "Pytest support for asyncio." -category = "dev" optional = false python-versions = ">= 3.5" files = [ @@ -1352,7 +1311,6 @@ testing = ["async_generator (>=1.3)", "coverage", "hypothesis (>=5.7.1)"] name = "pytest-cov" version = "2.12.1" description = "Pytest plugin for measuring coverage." -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -1372,7 +1330,6 @@ testing = ["fields", "hunter", "process-tests", "pytest-xdist", "six", "virtuale name = "pytest-mock" version = "3.10.0" description = "Thin-wrapper around the mock package for easier use with pytest" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1390,7 +1347,6 @@ dev = ["pre-commit", "pytest-asyncio", "tox"] name = "python-dateutil" version = "2.8.2" description = "Extensions to the standard Python datetime module" -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" files = [ @@ -1405,7 +1361,6 @@ six = ">=1.5" name = "pytz" version = "2023.3" description = "World timezone definitions, modern and historical" -category = "dev" optional = false python-versions = "*" files = [ @@ -1417,7 +1372,6 @@ files = [ name = "pyyaml" version = "6.0" description = "YAML parser and emitter for Python" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1467,7 +1421,6 @@ files = [ name = "requests" version = "2.31.0" description = "Python HTTP for Humans." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1489,7 +1442,6 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] name = "setuptools" version = "67.8.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1506,7 +1458,6 @@ testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs ( name = "six" version = "1.16.0" description = "Python 2 and 3 compatibility utilities" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -1518,7 +1469,6 @@ files = [ name = "snowballstemmer" version = "2.2.0" description = "This package provides 29 stemmers for 28 languages generated from Snowball algorithms." -category = "dev" optional = false python-versions = "*" files = [ @@ -1530,7 +1480,6 @@ files = [ name = "sphinx" version = "3.1.2" description = "Python documentation generator" -category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -1566,7 +1515,6 @@ test = ["cython", "html5lib", "pytest", "pytest-cov", "typed-ast"] name = "sphinx-rtd-theme" version = "0.5.0" description = "Read the Docs theme for Sphinx" -category = "dev" optional = false python-versions = "*" files = [ @@ -1584,7 +1532,6 @@ dev = ["bump2version", "sphinxcontrib-httpdomain", "transifex-client"] name = "sphinxcontrib-applehelp" version = "1.0.2" description = "sphinxcontrib-applehelp is a sphinx extension which outputs Apple help books" -category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -1600,7 +1547,6 @@ test = ["pytest"] name = "sphinxcontrib-devhelp" version = "1.0.2" description = "sphinxcontrib-devhelp is a sphinx extension which outputs Devhelp document." -category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -1616,7 +1562,6 @@ test = ["pytest"] name = "sphinxcontrib-htmlhelp" version = "2.0.0" description = "sphinxcontrib-htmlhelp is a sphinx extension which renders HTML help files" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1632,7 +1577,6 @@ test = ["html5lib", "pytest"] name = "sphinxcontrib-jsmath" version = "1.0.1" description = "A sphinx extension which renders display math in HTML via JavaScript" -category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -1647,7 +1591,6 @@ test = ["flake8", "mypy", "pytest"] name = "sphinxcontrib-qthelp" version = "1.0.3" description = "sphinxcontrib-qthelp is a sphinx extension which outputs QtHelp document." -category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -1663,7 +1606,6 @@ test = ["pytest"] name = "sphinxcontrib-serializinghtml" version = "1.1.5" description = "sphinxcontrib-serializinghtml is a sphinx extension which outputs \"serialized\" HTML files (json and pickle)." -category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -1679,7 +1621,6 @@ test = ["pytest"] name = "toml" version = "0.10.2" description = "Python Library for Tom's Obvious, Minimal Language" -category = "dev" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -1691,7 +1632,6 @@ files = [ name = "tomli" version = "2.0.1" description = "A lil' TOML parser" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1703,7 +1643,6 @@ files = [ name = "tomlkit" version = "0.7.2" description = "Style preserving TOML library" -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -1715,7 +1654,6 @@ files = [ name = "tox" version = "3.28.0" description = "tox is a generic virtualenv management and test command line tool" -category = "dev" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" files = [ @@ -1742,7 +1680,6 @@ testing = ["flaky (>=3.4.0)", "freezegun (>=0.3.11)", "pathlib2 (>=2.3.3)", "psu name = "typed-ast" version = "1.5.4" description = "a fork of Python 2 and 3 ast modules with type comment support" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1776,7 +1713,6 @@ files = [ name = "typing-extensions" version = "4.6.2" description = "Backported and Experimental Type Hints for Python 3.7+" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1788,7 +1724,6 @@ files = [ name = "urllib3" version = "2.0.2" description = "HTTP library with thread-safe connection pooling, file post, and more." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1806,7 +1741,6 @@ zstd = ["zstandard (>=0.18.0)"] name = "virtualenv" version = "20.23.0" description = "Virtual Python Environment builder" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1828,7 +1762,6 @@ test = ["covdefaults (>=2.3)", "coverage (>=7.2.3)", "coverage-enable-subprocess name = "wcwidth" version = "0.2.6" description = "Measures the displayed width of unicode strings in a terminal" -category = "dev" optional = false python-versions = "*" files = [ @@ -1840,7 +1773,6 @@ files = [ name = "zipp" version = "3.15.0" description = "Backport of pathlib-compatible object wrapper for zip files" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1858,4 +1790,4 @@ compiler = ["black", "isort", "jinja2"] [metadata] lock-version = "2.0" python-versions = "^3.7" -content-hash = "8f733a72705d31633a7f198a7a7dd6e3170876a1ccb8ca75b7d94b6379384a8f" +content-hash = "f3f3c42f9d20b60b53b7b4639dcd6ccfb945c63a55cb7485de72b5a298d0e618" diff --git a/pyproject.toml b/pyproject.toml index 4c43d7806..e6ccf37ec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ importlib-metadata = { version = ">=1.6.0", python = "<3.8" } jinja2 = { version = ">=3.0.3", optional = true } python-dateutil = "^2.8" isort = {version = "^5.11.5", optional = true} +betterproto-extras = { path = "betterproto-extras" } [tool.poetry.dev-dependencies] asv = "^0.4.2" diff --git a/src/betterproto/__init__.py b/src/betterproto/__init__.py index f02872701..2ca4b1a0c 100644 --- a/src/betterproto/__init__.py +++ b/src/betterproto/__init__.py @@ -1009,6 +1009,13 @@ def __bytes__(self) -> bytes: """ Get the binary encoded Protobuf representation of this message instance. """ + + if True: + # TODO: Make native serialization optional + + import betterproto_extras + return betterproto_extras.serialize(self) + with BytesIO() as stream: self.dump(stream) return stream.getvalue() @@ -1330,6 +1337,13 @@ def parse(self: T, data: "ReadableBuffer") -> T: :class:`Message` The initialized message. """ + if True: + # TODO: Make native deserialization optional + + import betterproto_extras + betterproto_extras.deserialize(self, data) + return self + with BytesIO(data) as stream: return self.load(stream)