From 9d85d492e8da9171f6dc287e918ec71be23ea002 Mon Sep 17 00:00:00 2001 From: Johannes Zottele Date: Thu, 17 Jul 2025 19:38:31 +0200 Subject: [PATCH 01/16] grpc-native: Add first protowire encoder/decoder Signed-off-by: Johannes Zottele --- grpc/grpc-core/build.gradle.kts | 18 +- .../nativeInterop/cinterop/libprotowire.def | 4 + .../{ => internal}/bridge/GrpcByteBuffer.kt | 2 +- .../grpc/{ => internal}/bridge/GrpcClient.kt | 2 +- .../grpc/{ => internal}/bridge/GrpcSlice.kt | 2 +- .../rpc/grpc/internal/protowire/KTag.kt | 34 + .../grpc/internal/protowire/WireDecoder.kt | 57 ++ .../grpc/internal/protowire/WireEncoder.kt | 40 ++ .../kotlin/kotlinx/rpc/grpc/BridgeTest.kt | 6 +- .../kotlin/kotlinx/rpc/grpc/ProtoWireTest.kt | 44 ++ grpc/grpcpp-c/BUILD.bazel | 20 + grpc/grpcpp-c/MODULE.bazel | 10 + grpc/grpcpp-c/MODULE.bazel.lock | 587 +++++++++++++++++- grpc/grpcpp-c/include/protowire.h | 39 ++ grpc/grpcpp-c/src/protowire.cpp | 95 +++ 15 files changed, 952 insertions(+), 8 deletions(-) create mode 100644 grpc/grpc-core/src/nativeInterop/cinterop/libprotowire.def rename grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/{ => internal}/bridge/GrpcByteBuffer.kt (95%) rename grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/{ => internal}/bridge/GrpcClient.kt (98%) rename grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/{ => internal}/bridge/GrpcSlice.kt (95%) create mode 100644 grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/protowire/KTag.kt create mode 100644 grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/protowire/WireDecoder.kt create mode 100644 grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/protowire/WireEncoder.kt create mode 100644 grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/ProtoWireTest.kt create mode 100644 grpc/grpcpp-c/include/protowire.h create mode 100644 grpc/grpcpp-c/src/protowire.cpp diff --git a/grpc/grpc-core/build.gradle.kts b/grpc/grpc-core/build.gradle.kts index a6f4f5803..0444a88b9 100644 --- a/grpc/grpc-core/build.gradle.kts +++ b/grpc/grpc-core/build.gradle.kts @@ -83,7 +83,7 @@ kotlin { val buildGrpcppCLib = tasks.register("buildGrpcppCLib") { group = "build" workingDir = grpcppCLib - commandLine("bash", "-c", "bazel build :grpcpp_c_static --config=release") + commandLine("bash", "-c", "bazel build :grpcpp_c_static :protowire_static --config=release") inputs.files(fileTree(grpcppCLib) { exclude("bazel-*/**") }) outputs.dir(grpcppCLib.resolve("bazel-bin")) @@ -108,6 +108,22 @@ kotlin { tasks.named(interopTask, CInteropProcess::class) { dependsOn(buildGrpcppCLib) } + + + val libprotowire by creating { + includeDirs( + grpcppCLib.resolve("include") + ) + extraOpts( + "-libraryPath", "${grpcppCLib.resolve("bazel-out/darwin_arm64-opt/bin")}", + ) + } + + val libUpbTask = "cinterop${libprotowire.name.capitalized()}${it.targetName.capitalized()}" + tasks.named(libUpbTask, CInteropProcess::class) { + dependsOn(buildGrpcppCLib) + } + } } } diff --git a/grpc/grpc-core/src/nativeInterop/cinterop/libprotowire.def b/grpc/grpc-core/src/nativeInterop/cinterop/libprotowire.def new file mode 100644 index 000000000..6cf4a33d5 --- /dev/null +++ b/grpc/grpc-core/src/nativeInterop/cinterop/libprotowire.def @@ -0,0 +1,4 @@ +headers = protowire.h +headerFilter = protowire.h + +staticLibraries = libprotowire_static.a \ No newline at end of file diff --git a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/bridge/GrpcByteBuffer.kt b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/bridge/GrpcByteBuffer.kt similarity index 95% rename from grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/bridge/GrpcByteBuffer.kt rename to grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/bridge/GrpcByteBuffer.kt index fb7317e97..7b4c72694 100644 --- a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/bridge/GrpcByteBuffer.kt +++ b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/bridge/GrpcByteBuffer.kt @@ -2,7 +2,7 @@ * Copyright 2023-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. */ -package kotlinx.rpc.grpc.bridge +package kotlinx.rpc.grpc.internal.bridge import kotlinx.cinterop.* import libgrpcpp_c.* diff --git a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/bridge/GrpcClient.kt b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/bridge/GrpcClient.kt similarity index 98% rename from grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/bridge/GrpcClient.kt rename to grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/bridge/GrpcClient.kt index f79764b87..891d81e79 100644 --- a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/bridge/GrpcClient.kt +++ b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/bridge/GrpcClient.kt @@ -2,7 +2,7 @@ * Copyright 2023-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. */ -package kotlinx.rpc.grpc.bridge +package kotlinx.rpc.grpc.internal.bridge import kotlinx.cinterop.* import kotlinx.coroutines.suspendCancellableCoroutine diff --git a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/bridge/GrpcSlice.kt b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/bridge/GrpcSlice.kt similarity index 95% rename from grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/bridge/GrpcSlice.kt rename to grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/bridge/GrpcSlice.kt index 9f9e06271..fa8738f0f 100644 --- a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/bridge/GrpcSlice.kt +++ b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/bridge/GrpcSlice.kt @@ -2,7 +2,7 @@ * Copyright 2023-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. */ -package kotlinx.rpc.grpc.bridge +package kotlinx.rpc.grpc.internal.bridge import kotlinx.cinterop.CValue import kotlinx.cinterop.ExperimentalForeignApi diff --git a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/protowire/KTag.kt b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/protowire/KTag.kt new file mode 100644 index 000000000..e6f789651 --- /dev/null +++ b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/protowire/KTag.kt @@ -0,0 +1,34 @@ +/* + * Copyright 2023-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.rpc.grpc.internal.protowire + +internal enum class WireType { + VARINT, // 0 + FIXED64, // 1 + LENGTH_DELIMITED, // 2 + START_GROUP, // 3 + END_GROUP, // 4 + FIXED32, // 5 +} + +internal data class KTag(val fieldNr: Int, val wireType: WireType) { + + + companion object { + // Number of bits in a tag which identify the wire type. + const val K_TAG_TYPE_BITS: UInt = 3u; + // Mask for those bits. (just 0b111) + val K_TAG_TYPE_MASK: UInt = (1u shl K_TAG_TYPE_BITS.toInt()) - 1u + } +} + +internal fun KTag.Companion.from(rawKTag: UInt): KTag? { + val type = rawKTag and K_TAG_TYPE_MASK + val field = rawKTag shr K_TAG_TYPE_BITS.toInt() + if (type >= WireType.entries.size.toUInt()) { + return null + } + return KTag(field.toInt(), WireType.entries[type.toInt()]) +} \ No newline at end of file diff --git a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/protowire/WireDecoder.kt b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/protowire/WireDecoder.kt new file mode 100644 index 000000000..65897937e --- /dev/null +++ b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/protowire/WireDecoder.kt @@ -0,0 +1,57 @@ +/* + * Copyright 2023-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.rpc.grpc.internal.protowire + +import kotlinx.cinterop.* +import libprotowire.pw_decoder_delete +import libprotowire.pw_decoder_delete_opaque_string +import libprotowire.pw_decoder_new +import libprotowire.pw_decoder_read_bool +import libprotowire.pw_decoder_read_string +import libprotowire.pw_decoder_read_tag +import kotlin.experimental.ExperimentalNativeApi +import kotlin.native.ref.createCleaner + +@ExperimentalForeignApi +@OptIn(ExperimentalNativeApi::class) +internal class WireDecoder(buffer: UByteArray) { + internal val pinnedBuffer = buffer.pin() + internal val raw = pw_decoder_new(pinnedBuffer.addressOf(0), buffer.size.toUInt()) + ?: error("Failed to create proto wire decoder") + + init { + // free the encoder once garbage collector collects its pointer + createCleaner(raw) { + pw_decoder_delete(it) + } + } + + fun readTag(): KTag? { + val tag = pw_decoder_read_tag(raw) + return KTag.from(tag) + } + + fun readBool(): Boolean? = memScoped { + val bool = alloc() + if (pw_decoder_read_bool(raw, bool.ptr)) { + return bool.value + } + return null + } + + fun readString(): String? = memScoped { + // allocate an opaque pointer that holds a reference to the allocated std::string. + // by keeping std::string object alive, we avoid the need to copy the C++ string to an allocated C string. + val opaqueStr = alloc() + val cCharPtr = alloc>() + val ok = pw_decoder_read_string(raw, opaqueStr.ptr, cCharPtr.ptr) + var result: String? = null + if (ok) result = cCharPtr.value?.toKString() + // after copying the string to a Kotlin string, we must delete the allocated C++ string (includes the C string) + pw_decoder_delete_opaque_string(opaqueStr.value) + return result; + + } +} diff --git a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/protowire/WireEncoder.kt b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/protowire/WireEncoder.kt new file mode 100644 index 000000000..1c54d7d24 --- /dev/null +++ b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/protowire/WireEncoder.kt @@ -0,0 +1,40 @@ +/* + * Copyright 2023-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.rpc.grpc.internal.protowire + +import kotlinx.cinterop.ExperimentalForeignApi +import kotlinx.cinterop.Pinned +import kotlinx.cinterop.addressOf +import kotlinx.cinterop.pin +import libprotowire.pw_encoder_delete +import libprotowire.pw_encoder_new +import libprotowire.pw_encoder_write_bool +import libprotowire.pw_encoder_write_string +import kotlin.experimental.ExperimentalNativeApi +import kotlin.native.ref.createCleaner + +@ExperimentalForeignApi +@OptIn(ExperimentalNativeApi::class) +internal class WireEncoder { + internal val buffer: Pinned = UByteArray(1024).pin() + internal val raw = pw_encoder_new(buffer.addressOf(0), 1024u) + ?: error("Failed to create proto wire encoder") + + init { + // free the encoder once garbage collector collects its pointer + createCleaner(raw) { + pw_encoder_delete(it) + } + } + + fun write(field: Int, value: Boolean): Boolean { + return pw_encoder_write_bool(raw, field, value) + } + + fun write(field: Int, value: String): Boolean { + return pw_encoder_write_string(raw, field, value) + } + +} \ No newline at end of file diff --git a/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/BridgeTest.kt b/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/BridgeTest.kt index ead336b5f..3a9a40e9c 100644 --- a/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/BridgeTest.kt +++ b/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/BridgeTest.kt @@ -5,9 +5,9 @@ package kotlinx.rpc.grpc import kotlinx.coroutines.runBlocking -import kotlinx.rpc.grpc.bridge.GrpcByteBuffer -import kotlinx.rpc.grpc.bridge.GrpcClient -import kotlinx.rpc.grpc.bridge.GrpcSlice +import kotlinx.rpc.grpc.internal.bridge.GrpcByteBuffer +import kotlinx.rpc.grpc.internal.bridge.GrpcClient +import kotlinx.rpc.grpc.internal.bridge.GrpcSlice import libgrpcpp_c.pb_decode_greeter_sayhello_response import kotlin.test.Test diff --git a/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/ProtoWireTest.kt b/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/ProtoWireTest.kt new file mode 100644 index 000000000..677088633 --- /dev/null +++ b/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/ProtoWireTest.kt @@ -0,0 +1,44 @@ +/* + * Copyright 2023-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.rpc.grpc + +import kotlinx.cinterop.ExperimentalForeignApi +import kotlinx.rpc.grpc.internal.protowire.WireDecoder +import kotlinx.rpc.grpc.internal.protowire.WireEncoder +import kotlinx.rpc.grpc.internal.protowire.WireType +import kotlin.test.Test + +@OptIn(ExperimentalForeignApi::class) +class ProtoWireTest { + + @Test + fun testEncodeDecodeBool() { + + val fieldNr = 3; + + val encoder = WireEncoder() + encoder.write(fieldNr, true) + encoder.write(4, "Hello Test") + + val encodedBuffer = encoder.buffer.get() + val decoder = WireDecoder(encodedBuffer) + + val t1 = decoder.readTag()!! + check(t1.wireType == WireType.VARINT) + check(t1.fieldNr == fieldNr) + + val bool = decoder.readBool()!! + check(bool) + + val t2 = decoder.readTag()!! + check(t2.wireType == WireType.LENGTH_DELIMITED) + check(t2.fieldNr == 4) + + val str = decoder.readString()!! + check(str == "Hello Test") + + + } +} diff --git a/grpc/grpcpp-c/BUILD.bazel b/grpc/grpcpp-c/BUILD.bazel index 9bdb4e927..0cce2f26a 100644 --- a/grpc/grpcpp-c/BUILD.bazel +++ b/grpc/grpcpp-c/BUILD.bazel @@ -1,3 +1,4 @@ +load("@emsdk//emscripten_toolchain:wasm_rules.bzl", "wasm_cc_binary") load("@rules_cc//cc:defs.bzl", "cc_library") cc_library( @@ -13,6 +14,25 @@ cc_library( ], ) +cc_library( + name = "protowire", + srcs = ["src/protowire.cpp"], + hdrs = glob(["include/*.h"]), + copts = ["-std=c++20"], + includes = ["include"], + visibility = ["//visibility:public"], + deps = [ + "@com_google_protobuf//:protobuf_lite", + ], +) + +cc_static_library( + name = "protowire_static", + deps = [ + ":protowire", + ], +) + cc_static_library( name = "grpcpp_c_static", deps = [ diff --git a/grpc/grpcpp-c/MODULE.bazel b/grpc/grpcpp-c/MODULE.bazel index 136cad3ae..b729c8a44 100644 --- a/grpc/grpcpp-c/MODULE.bazel +++ b/grpc/grpcpp-c/MODULE.bazel @@ -22,3 +22,13 @@ bazel_dep( version = "1.73.1", repo_name = "com_github_grpc_grpc", ) + +emsdk_version = "4.0.11" + +bazel_dep(name = "emsdk", version = emsdk_version) +git_override( + module_name = "emsdk", + remote = "https://github.com/emscripten-core/emsdk.git", + strip_prefix = "bazel", + tag = emsdk_version, +) diff --git a/grpc/grpcpp-c/MODULE.bazel.lock b/grpc/grpcpp-c/MODULE.bazel.lock index 09e36896c..9e94f9a39 100644 --- a/grpc/grpcpp-c/MODULE.bazel.lock +++ b/grpc/grpcpp-c/MODULE.bazel.lock @@ -24,8 +24,12 @@ "https://bcr.bazel.build/modules/aspect_bazel_lib/1.31.2/MODULE.bazel": "7bee702b4862612f29333590f4b658a5832d433d6f8e4395f090e8f4e85d442f", "https://bcr.bazel.build/modules/aspect_bazel_lib/1.38.0/MODULE.bazel": "6307fec451ba9962c1c969eb516ebfe1e46528f7fa92e1c9ac8646bef4cdaa3f", "https://bcr.bazel.build/modules/aspect_bazel_lib/1.40.3/MODULE.bazel": "668e6bcb4d957fc0e284316dba546b705c8d43c857f87119619ee83c4555b859", + "https://bcr.bazel.build/modules/aspect_bazel_lib/1.42.3/MODULE.bazel": "e4529e12d8cd5b828e2b5960d07d3ec032541740d419d7d5b859cabbf5b056f9", + "https://bcr.bazel.build/modules/aspect_bazel_lib/1.42.3/source.json": "80cb66069ad626e0921555cd2bf278286fd7763fae2450e564e351792e8303f4", "https://bcr.bazel.build/modules/aspect_rules_js/1.33.1/MODULE.bazel": "db3e7f16e471cf6827059d03af7c21859e7a0d2bc65429a3a11f005d46fc501b", "https://bcr.bazel.build/modules/aspect_rules_js/1.39.0/MODULE.bazel": "aece421d479e3c31dc3e5f6d49a12acc2700457c03c556650ec7a0ff23fc0d95", + "https://bcr.bazel.build/modules/aspect_rules_js/1.42.0/MODULE.bazel": "f19e6b4a16f77f8cf3728eac1f60dbfd8e043517fd4f4dbf17a75a6c50936d62", + "https://bcr.bazel.build/modules/aspect_rules_js/1.42.0/source.json": "abbb3eac3b6af76b8ce230a9a901c6d08d93f4f5ffd55314bf630827dddee57e", "https://bcr.bazel.build/modules/aspect_rules_lint/0.12.0/MODULE.bazel": "e767c5dbfeb254ec03275a7701b5cfde2c4d2873676804bc7cb27ddff3728fed", "https://bcr.bazel.build/modules/bazel_features/0.1.0/MODULE.bazel": "47011d645b0f949f42ee67f2e8775188a9cf4a0a1528aa2fa4952f2fd00906fd", "https://bcr.bazel.build/modules/bazel_features/1.1.0/MODULE.bazel": "cfd42ff3b815a5f39554d97182657f8c4b9719568eb7fded2b9135f084bf760b", @@ -42,6 +46,7 @@ "https://bcr.bazel.build/modules/bazel_features/1.30.0/MODULE.bazel": "a14b62d05969a293b80257e72e597c2da7f717e1e69fa8b339703ed6731bec87", "https://bcr.bazel.build/modules/bazel_features/1.30.0/source.json": "b07e17f067fe4f69f90b03b36ef1e08fe0d1f3cac254c1241a1818773e3423bc", "https://bcr.bazel.build/modules/bazel_features/1.4.1/MODULE.bazel": "e45b6bb2350aff3e442ae1111c555e27eac1d915e77775f6fdc4b351b758b5d7", + "https://bcr.bazel.build/modules/bazel_features/1.9.0/MODULE.bazel": "885151d58d90d8d9c811eb75e3288c11f850e1d6b481a8c9f766adee4712358b", "https://bcr.bazel.build/modules/bazel_features/1.9.1/MODULE.bazel": "8f679097876a9b609ad1f60249c49d68bfab783dd9be012faf9d82547b14815a", "https://bcr.bazel.build/modules/bazel_skylib/1.0.3/MODULE.bazel": "bcb0fd896384802d1ad283b4e4eb4d718eebd8cb820b0a2c3a347fb971afd9d8", "https://bcr.bazel.build/modules/bazel_skylib/1.1.1/MODULE.bazel": "1add3e7d93ff2e6998f9e118022c84d163917d912f5afafb3058e3d2f1545b5e", @@ -254,6 +259,8 @@ "https://bcr.bazel.build/modules/rules_license/1.0.0/MODULE.bazel": "a7fda60eefdf3d8c827262ba499957e4df06f659330bbe6cdbdb975b768bb65c", "https://bcr.bazel.build/modules/rules_license/1.0.0/source.json": "a52c89e54cc311196e478f8382df91c15f7a2bfdf4c6cd0e2675cc2ff0b56efb", "https://bcr.bazel.build/modules/rules_nodejs/5.8.2/MODULE.bazel": "6bc03c8f37f69401b888023bf511cb6ee4781433b0cb56236b2e55a21e3a026a", + "https://bcr.bazel.build/modules/rules_nodejs/6.3.2/MODULE.bazel": "42e8d5254b6135f890fecca7c8d7f95a7d27a45f8275b276f66ec337767530ef", + "https://bcr.bazel.build/modules/rules_nodejs/6.3.2/source.json": "80e0a68eb81772f1631f8b69014884eebc2474b3b3025fd19a5240ae4f76f9c9", "https://bcr.bazel.build/modules/rules_perl/0.2.4/MODULE.bazel": "5f5af7be4bf5fb88d91af7469518f0fd2161718aefc606188f7cd51f436ca938", "https://bcr.bazel.build/modules/rules_perl/0.2.4/source.json": "574317d6b3c7e4843fe611b76f15e62a1889949f5570702e1ee4ad335ea3c339", "https://bcr.bazel.build/modules/rules_pkg/0.7.0/MODULE.bazel": "df99f03fc7934a4737122518bb87e667e62d780b610910f0447665a7e2be62dc", @@ -280,7 +287,8 @@ "https://bcr.bazel.build/modules/rules_python/0.4.0/MODULE.bazel": "9208ee05fd48bf09ac60ed269791cf17fb343db56c8226a720fbb1cdf467166c", "https://bcr.bazel.build/modules/rules_python/0.40.0/MODULE.bazel": "9d1a3cd88ed7d8e39583d9ffe56ae8a244f67783ae89b60caafc9f5cf318ada7", "https://bcr.bazel.build/modules/rules_python/1.0.0/MODULE.bazel": "898a3d999c22caa585eb062b600f88654bf92efb204fa346fb55f6f8edffca43", - "https://bcr.bazel.build/modules/rules_python/1.0.0/source.json": "b0162a65c6312e45e7912e39abd1a7f8856c2c7e41ecc9b6dc688a6f6400a917", + "https://bcr.bazel.build/modules/rules_python/1.3.0/MODULE.bazel": "8361d57eafb67c09b75bf4bbe6be360e1b8f4f18118ab48037f2bd50aa2ccb13", + "https://bcr.bazel.build/modules/rules_python/1.3.0/source.json": "25932f917cd279c7baefa6cb1d3fa8750a7a29de522024449b19af6eab51f4a0", "https://bcr.bazel.build/modules/rules_rust/0.51.0/MODULE.bazel": "2b6d1617ac8503bfdcc0e4520c20539d4bba3a691100bee01afe193ceb0310f9", "https://bcr.bazel.build/modules/rules_shell/0.2.0/MODULE.bazel": "fda8a652ab3c7d8fee214de05e7a9916d8b28082234e8d2c0094505c5268ed3c", "https://bcr.bazel.build/modules/rules_shell/0.3.0/MODULE.bazel": "de4402cd12f4cc8fda2354fce179fdb068c0b9ca1ec2d2b17b3e21b24c1a937b", @@ -350,6 +358,426 @@ ] } }, + "@@aspect_bazel_lib+//lib:extensions.bzl%toolchains": { + "general": { + "bzlTransitiveDigest": "FRH/uLcAIxs4VtonQvNHnu3yGF1glDBtm+FyaO6lOI4=", + "usagesDigest": "1c7PNX163TGNqWzfejRnWpH/hiT4/GRG0kYxuez0Uz0=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "copy_directory_darwin_amd64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:copy_directory_toolchain.bzl%copy_directory_platform_repo", + "attributes": { + "platform": "darwin_amd64" + } + }, + "copy_directory_darwin_arm64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:copy_directory_toolchain.bzl%copy_directory_platform_repo", + "attributes": { + "platform": "darwin_arm64" + } + }, + "copy_directory_freebsd_amd64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:copy_directory_toolchain.bzl%copy_directory_platform_repo", + "attributes": { + "platform": "freebsd_amd64" + } + }, + "copy_directory_linux_amd64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:copy_directory_toolchain.bzl%copy_directory_platform_repo", + "attributes": { + "platform": "linux_amd64" + } + }, + "copy_directory_linux_arm64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:copy_directory_toolchain.bzl%copy_directory_platform_repo", + "attributes": { + "platform": "linux_arm64" + } + }, + "copy_directory_windows_amd64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:copy_directory_toolchain.bzl%copy_directory_platform_repo", + "attributes": { + "platform": "windows_amd64" + } + }, + "copy_directory_toolchains": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:copy_directory_toolchain.bzl%copy_directory_toolchains_repo", + "attributes": { + "user_repository_name": "copy_directory" + } + }, + "copy_to_directory_darwin_amd64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:copy_to_directory_toolchain.bzl%copy_to_directory_platform_repo", + "attributes": { + "platform": "darwin_amd64" + } + }, + "copy_to_directory_darwin_arm64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:copy_to_directory_toolchain.bzl%copy_to_directory_platform_repo", + "attributes": { + "platform": "darwin_arm64" + } + }, + "copy_to_directory_freebsd_amd64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:copy_to_directory_toolchain.bzl%copy_to_directory_platform_repo", + "attributes": { + "platform": "freebsd_amd64" + } + }, + "copy_to_directory_linux_amd64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:copy_to_directory_toolchain.bzl%copy_to_directory_platform_repo", + "attributes": { + "platform": "linux_amd64" + } + }, + "copy_to_directory_linux_arm64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:copy_to_directory_toolchain.bzl%copy_to_directory_platform_repo", + "attributes": { + "platform": "linux_arm64" + } + }, + "copy_to_directory_windows_amd64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:copy_to_directory_toolchain.bzl%copy_to_directory_platform_repo", + "attributes": { + "platform": "windows_amd64" + } + }, + "copy_to_directory_toolchains": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:copy_to_directory_toolchain.bzl%copy_to_directory_toolchains_repo", + "attributes": { + "user_repository_name": "copy_to_directory" + } + }, + "jq_darwin_amd64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:jq_toolchain.bzl%jq_platform_repo", + "attributes": { + "platform": "darwin_amd64", + "version": "1.6" + } + }, + "jq_darwin_arm64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:jq_toolchain.bzl%jq_platform_repo", + "attributes": { + "platform": "darwin_arm64", + "version": "1.6" + } + }, + "jq_linux_amd64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:jq_toolchain.bzl%jq_platform_repo", + "attributes": { + "platform": "linux_amd64", + "version": "1.6" + } + }, + "jq_windows_amd64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:jq_toolchain.bzl%jq_platform_repo", + "attributes": { + "platform": "windows_amd64", + "version": "1.6" + } + }, + "jq": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:jq_toolchain.bzl%jq_host_alias_repo", + "attributes": {} + }, + "jq_toolchains": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:jq_toolchain.bzl%jq_toolchains_repo", + "attributes": { + "user_repository_name": "jq" + } + }, + "yq_darwin_amd64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:yq_toolchain.bzl%yq_platform_repo", + "attributes": { + "platform": "darwin_amd64", + "version": "4.25.2" + } + }, + "yq_darwin_arm64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:yq_toolchain.bzl%yq_platform_repo", + "attributes": { + "platform": "darwin_arm64", + "version": "4.25.2" + } + }, + "yq_linux_amd64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:yq_toolchain.bzl%yq_platform_repo", + "attributes": { + "platform": "linux_amd64", + "version": "4.25.2" + } + }, + "yq_linux_arm64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:yq_toolchain.bzl%yq_platform_repo", + "attributes": { + "platform": "linux_arm64", + "version": "4.25.2" + } + }, + "yq_linux_s390x": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:yq_toolchain.bzl%yq_platform_repo", + "attributes": { + "platform": "linux_s390x", + "version": "4.25.2" + } + }, + "yq_linux_ppc64le": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:yq_toolchain.bzl%yq_platform_repo", + "attributes": { + "platform": "linux_ppc64le", + "version": "4.25.2" + } + }, + "yq_windows_amd64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:yq_toolchain.bzl%yq_platform_repo", + "attributes": { + "platform": "windows_amd64", + "version": "4.25.2" + } + }, + "yq": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:yq_toolchain.bzl%yq_host_alias_repo", + "attributes": {} + }, + "yq_toolchains": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:yq_toolchain.bzl%yq_toolchains_repo", + "attributes": { + "user_repository_name": "yq" + } + }, + "coreutils_darwin_amd64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:coreutils_toolchain.bzl%coreutils_platform_repo", + "attributes": { + "platform": "darwin_amd64", + "version": "0.0.16" + } + }, + "coreutils_darwin_arm64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:coreutils_toolchain.bzl%coreutils_platform_repo", + "attributes": { + "platform": "darwin_arm64", + "version": "0.0.16" + } + }, + "coreutils_linux_amd64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:coreutils_toolchain.bzl%coreutils_platform_repo", + "attributes": { + "platform": "linux_amd64", + "version": "0.0.16" + } + }, + "coreutils_linux_arm64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:coreutils_toolchain.bzl%coreutils_platform_repo", + "attributes": { + "platform": "linux_arm64", + "version": "0.0.16" + } + }, + "coreutils_windows_amd64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:coreutils_toolchain.bzl%coreutils_platform_repo", + "attributes": { + "platform": "windows_amd64", + "version": "0.0.16" + } + }, + "coreutils_toolchains": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:coreutils_toolchain.bzl%coreutils_toolchains_repo", + "attributes": { + "user_repository_name": "coreutils" + } + }, + "bsd_tar_darwin_amd64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:tar_toolchain.bzl%bsdtar_binary_repo", + "attributes": { + "platform": "darwin_amd64" + } + }, + "bsd_tar_darwin_arm64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:tar_toolchain.bzl%bsdtar_binary_repo", + "attributes": { + "platform": "darwin_arm64" + } + }, + "bsd_tar_linux_amd64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:tar_toolchain.bzl%bsdtar_binary_repo", + "attributes": { + "platform": "linux_amd64" + } + }, + "bsd_tar_linux_arm64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:tar_toolchain.bzl%bsdtar_binary_repo", + "attributes": { + "platform": "linux_arm64" + } + }, + "bsd_tar_windows_amd64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:tar_toolchain.bzl%bsdtar_binary_repo", + "attributes": { + "platform": "windows_amd64" + } + }, + "bsd_tar_toolchains": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:tar_toolchain.bzl%tar_toolchains_repo", + "attributes": { + "user_repository_name": "bsd_tar" + } + }, + "expand_template_darwin_amd64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:expand_template_toolchain.bzl%expand_template_platform_repo", + "attributes": { + "platform": "darwin_amd64" + } + }, + "expand_template_darwin_arm64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:expand_template_toolchain.bzl%expand_template_platform_repo", + "attributes": { + "platform": "darwin_arm64" + } + }, + "expand_template_freebsd_amd64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:expand_template_toolchain.bzl%expand_template_platform_repo", + "attributes": { + "platform": "freebsd_amd64" + } + }, + "expand_template_linux_amd64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:expand_template_toolchain.bzl%expand_template_platform_repo", + "attributes": { + "platform": "linux_amd64" + } + }, + "expand_template_linux_arm64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:expand_template_toolchain.bzl%expand_template_platform_repo", + "attributes": { + "platform": "linux_arm64" + } + }, + "expand_template_windows_amd64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:expand_template_toolchain.bzl%expand_template_platform_repo", + "attributes": { + "platform": "windows_amd64" + } + }, + "expand_template_toolchains": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:expand_template_toolchain.bzl%expand_template_toolchains_repo", + "attributes": { + "user_repository_name": "expand_template" + } + } + }, + "recordedRepoMappingEntries": [ + [ + "aspect_bazel_lib+", + "aspect_bazel_lib", + "aspect_bazel_lib+" + ], + [ + "aspect_bazel_lib+", + "bazel_skylib", + "bazel_skylib+" + ], + [ + "aspect_bazel_lib+", + "bazel_tools", + "bazel_tools" + ] + ] + } + }, + "@@emsdk+//:emscripten_cache.bzl%emscripten_cache": { + "general": { + "bzlTransitiveDigest": "uqDvXmpTNqW4+ie/Fk+xC3TrFrKvL+9hNtoP51Kt2oo=", + "usagesDigest": "iaw2BH+XNky0wzaCMCWcOxr/wRXVypwc4a22UDwIjIs=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "emscripten_cache": { + "repoRuleId": "@@emsdk+//:emscripten_cache.bzl%_emscripten_cache_repository", + "attributes": { + "configuration": [], + "targets": [] + } + } + }, + "recordedRepoMappingEntries": [] + } + }, + "@@emsdk+//:emscripten_deps.bzl%emscripten_deps": { + "general": { + "bzlTransitiveDigest": "hjVXd1Th8cZxk9rw2px6WtJqoM900gx6e4EEMalHmis=", + "usagesDigest": "hZ+VngAMPf2nklmauHosp51Gh/R1WN3gtTJ/lVPCmjg=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "emscripten_bin_linux": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "build_file_content": "\npackage(default_visibility = ['//visibility:public'])\n\nfilegroup(\n name = \"all\",\n srcs = glob([\"**\"]),\n)\n\nfilegroup(\n name = \"includes\",\n srcs = glob([\n \"emscripten/cache/sysroot/include/c++/v1/**\",\n \"emscripten/cache/sysroot/include/compat/**\",\n \"emscripten/cache/sysroot/include/**\",\n \"lib/clang/**/include/**\",\n ]),\n)\n\nfilegroup(\n name = \"emcc_common\",\n srcs = [\n \"emscripten/emcc.py\",\n \"emscripten/embuilder.py\",\n \"emscripten/emscripten-version.txt\",\n \"emscripten/cache/sysroot_install.stamp\",\n \"emscripten/src/settings.js\",\n \"emscripten/src/settings_internal.js\",\n ] + glob(\n include = [\n \"emscripten/third_party/**\",\n \"emscripten/tools/**\",\n ],\n exclude = [\n \"**/__pycache__/**\",\n ],\n ),\n)\n\nfilegroup(\n name = \"compiler_files\",\n srcs = [\n \"bin/clang\",\n \"bin/clang++\",\n \":emcc_common\",\n \":includes\",\n ],\n)\n\nfilegroup(\n name = \"linker_files\",\n srcs = [\n \"bin/clang\",\n \"bin/llvm-ar\",\n \"bin/llvm-dwarfdump\",\n \"bin/llvm-nm\",\n \"bin/llvm-objcopy\",\n \"bin/wasm-ctor-eval\",\n \"bin/wasm-emscripten-finalize\",\n \"bin/wasm-ld\",\n \"bin/wasm-metadce\",\n \"bin/wasm-opt\",\n \"bin/wasm-split\",\n \"bin/wasm2js\",\n \":emcc_common\",\n ] + glob(\n include = [\n \"emscripten/cache/sysroot/lib/**\",\n \"emscripten/node_modules/**\",\n \"emscripten/src/**\",\n ],\n ),\n)\n\nfilegroup(\n name = \"ar_files\",\n srcs = [\n \"bin/llvm-ar\",\n \"emscripten/emar.py\",\n \"emscripten/emscripten-version.txt\",\n \"emscripten/src/settings.js\",\n \"emscripten/src/settings_internal.js\",\n ] + glob(\n include = [\n \"emscripten/tools/**\",\n ],\n exclude = [\n \"**/__pycache__/**\",\n ],\n ),\n)\n", + "sha256": "f38e70b53be587e7c757f375b3452e259c70130d4b40db3213c95b7ae321f5d7", + "strip_prefix": "install", + "type": "tar.xz", + "url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/linux/7033fec38817ec01909b044ea0193ddd5057255c/wasm-binaries.tar.xz" + } + }, + "emscripten_bin_linux_arm64": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "build_file_content": "\npackage(default_visibility = ['//visibility:public'])\n\nfilegroup(\n name = \"all\",\n srcs = glob([\"**\"]),\n)\n\nfilegroup(\n name = \"includes\",\n srcs = glob([\n \"emscripten/cache/sysroot/include/c++/v1/**\",\n \"emscripten/cache/sysroot/include/compat/**\",\n \"emscripten/cache/sysroot/include/**\",\n \"lib/clang/**/include/**\",\n ]),\n)\n\nfilegroup(\n name = \"emcc_common\",\n srcs = [\n \"emscripten/emcc.py\",\n \"emscripten/embuilder.py\",\n \"emscripten/emscripten-version.txt\",\n \"emscripten/cache/sysroot_install.stamp\",\n \"emscripten/src/settings.js\",\n \"emscripten/src/settings_internal.js\",\n ] + glob(\n include = [\n \"emscripten/third_party/**\",\n \"emscripten/tools/**\",\n ],\n exclude = [\n \"**/__pycache__/**\",\n ],\n ),\n)\n\nfilegroup(\n name = \"compiler_files\",\n srcs = [\n \"bin/clang\",\n \"bin/clang++\",\n \":emcc_common\",\n \":includes\",\n ],\n)\n\nfilegroup(\n name = \"linker_files\",\n srcs = [\n \"bin/clang\",\n \"bin/llvm-ar\",\n \"bin/llvm-dwarfdump\",\n \"bin/llvm-nm\",\n \"bin/llvm-objcopy\",\n \"bin/wasm-ctor-eval\",\n \"bin/wasm-emscripten-finalize\",\n \"bin/wasm-ld\",\n \"bin/wasm-metadce\",\n \"bin/wasm-opt\",\n \"bin/wasm-split\",\n \"bin/wasm2js\",\n \":emcc_common\",\n ] + glob(\n include = [\n \"emscripten/cache/sysroot/lib/**\",\n \"emscripten/node_modules/**\",\n \"emscripten/src/**\",\n ],\n ),\n)\n\nfilegroup(\n name = \"ar_files\",\n srcs = [\n \"bin/llvm-ar\",\n \"emscripten/emar.py\",\n \"emscripten/emscripten-version.txt\",\n \"emscripten/src/settings.js\",\n \"emscripten/src/settings_internal.js\",\n ] + glob(\n include = [\n \"emscripten/tools/**\",\n ],\n exclude = [\n \"**/__pycache__/**\",\n ],\n ),\n)\n", + "sha256": "42020e4db200ac366a3e91ac2fccc04ee0ffc090cd2d5986c892b27f39172bb9", + "strip_prefix": "install", + "type": "tar.xz", + "url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/linux/7033fec38817ec01909b044ea0193ddd5057255c/wasm-binaries-arm64.tar.xz" + } + }, + "emscripten_bin_mac": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "build_file_content": "\npackage(default_visibility = ['//visibility:public'])\n\nfilegroup(\n name = \"all\",\n srcs = glob([\"**\"]),\n)\n\nfilegroup(\n name = \"includes\",\n srcs = glob([\n \"emscripten/cache/sysroot/include/c++/v1/**\",\n \"emscripten/cache/sysroot/include/compat/**\",\n \"emscripten/cache/sysroot/include/**\",\n \"lib/clang/**/include/**\",\n ]),\n)\n\nfilegroup(\n name = \"emcc_common\",\n srcs = [\n \"emscripten/emcc.py\",\n \"emscripten/embuilder.py\",\n \"emscripten/emscripten-version.txt\",\n \"emscripten/cache/sysroot_install.stamp\",\n \"emscripten/src/settings.js\",\n \"emscripten/src/settings_internal.js\",\n ] + glob(\n include = [\n \"emscripten/third_party/**\",\n \"emscripten/tools/**\",\n ],\n exclude = [\n \"**/__pycache__/**\",\n ],\n ),\n)\n\nfilegroup(\n name = \"compiler_files\",\n srcs = [\n \"bin/clang\",\n \"bin/clang++\",\n \":emcc_common\",\n \":includes\",\n ],\n)\n\nfilegroup(\n name = \"linker_files\",\n srcs = [\n \"bin/clang\",\n \"bin/llvm-ar\",\n \"bin/llvm-dwarfdump\",\n \"bin/llvm-nm\",\n \"bin/llvm-objcopy\",\n \"bin/wasm-ctor-eval\",\n \"bin/wasm-emscripten-finalize\",\n \"bin/wasm-ld\",\n \"bin/wasm-metadce\",\n \"bin/wasm-opt\",\n \"bin/wasm-split\",\n \"bin/wasm2js\",\n \":emcc_common\",\n ] + glob(\n include = [\n \"emscripten/cache/sysroot/lib/**\",\n \"emscripten/node_modules/**\",\n \"emscripten/src/**\",\n ],\n ),\n)\n\nfilegroup(\n name = \"ar_files\",\n srcs = [\n \"bin/llvm-ar\",\n \"emscripten/emar.py\",\n \"emscripten/emscripten-version.txt\",\n \"emscripten/src/settings.js\",\n \"emscripten/src/settings_internal.js\",\n ] + glob(\n include = [\n \"emscripten/tools/**\",\n ],\n exclude = [\n \"**/__pycache__/**\",\n ],\n ),\n)\n", + "sha256": "4169811f9682f54ae5c9d0662d0a4dd4318abab5d3d0473fa54007f515a8cdea", + "strip_prefix": "install", + "type": "tar.xz", + "url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/mac/7033fec38817ec01909b044ea0193ddd5057255c/wasm-binaries.tar.xz" + } + }, + "emscripten_bin_mac_arm64": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "build_file_content": "\npackage(default_visibility = ['//visibility:public'])\n\nfilegroup(\n name = \"all\",\n srcs = glob([\"**\"]),\n)\n\nfilegroup(\n name = \"includes\",\n srcs = glob([\n \"emscripten/cache/sysroot/include/c++/v1/**\",\n \"emscripten/cache/sysroot/include/compat/**\",\n \"emscripten/cache/sysroot/include/**\",\n \"lib/clang/**/include/**\",\n ]),\n)\n\nfilegroup(\n name = \"emcc_common\",\n srcs = [\n \"emscripten/emcc.py\",\n \"emscripten/embuilder.py\",\n \"emscripten/emscripten-version.txt\",\n \"emscripten/cache/sysroot_install.stamp\",\n \"emscripten/src/settings.js\",\n \"emscripten/src/settings_internal.js\",\n ] + glob(\n include = [\n \"emscripten/third_party/**\",\n \"emscripten/tools/**\",\n ],\n exclude = [\n \"**/__pycache__/**\",\n ],\n ),\n)\n\nfilegroup(\n name = \"compiler_files\",\n srcs = [\n \"bin/clang\",\n \"bin/clang++\",\n \":emcc_common\",\n \":includes\",\n ],\n)\n\nfilegroup(\n name = \"linker_files\",\n srcs = [\n \"bin/clang\",\n \"bin/llvm-ar\",\n \"bin/llvm-dwarfdump\",\n \"bin/llvm-nm\",\n \"bin/llvm-objcopy\",\n \"bin/wasm-ctor-eval\",\n \"bin/wasm-emscripten-finalize\",\n \"bin/wasm-ld\",\n \"bin/wasm-metadce\",\n \"bin/wasm-opt\",\n \"bin/wasm-split\",\n \"bin/wasm2js\",\n \":emcc_common\",\n ] + glob(\n include = [\n \"emscripten/cache/sysroot/lib/**\",\n \"emscripten/node_modules/**\",\n \"emscripten/src/**\",\n ],\n ),\n)\n\nfilegroup(\n name = \"ar_files\",\n srcs = [\n \"bin/llvm-ar\",\n \"emscripten/emar.py\",\n \"emscripten/emscripten-version.txt\",\n \"emscripten/src/settings.js\",\n \"emscripten/src/settings_internal.js\",\n ] + glob(\n include = [\n \"emscripten/tools/**\",\n ],\n exclude = [\n \"**/__pycache__/**\",\n ],\n ),\n)\n", + "sha256": "09554371e3941306d047d67618532e5366ba6c9b5bda1a504a917bfbabc5d414", + "strip_prefix": "install", + "type": "tar.xz", + "url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/mac/7033fec38817ec01909b044ea0193ddd5057255c/wasm-binaries-arm64.tar.xz" + } + }, + "emscripten_bin_win": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "build_file_content": "\npackage(default_visibility = ['//visibility:public'])\n\nfilegroup(\n name = \"all\",\n srcs = glob([\"**\"]),\n)\n\nfilegroup(\n name = \"includes\",\n srcs = glob([\n \"emscripten/cache/sysroot/include/c++/v1/**\",\n \"emscripten/cache/sysroot/include/compat/**\",\n \"emscripten/cache/sysroot/include/**\",\n \"lib/clang/**/include/**\",\n ]),\n)\n\nfilegroup(\n name = \"emcc_common\",\n srcs = [\n \"emscripten/emcc.py\",\n \"emscripten/embuilder.py\",\n \"emscripten/emscripten-version.txt\",\n \"emscripten/cache/sysroot_install.stamp\",\n \"emscripten/src/settings.js\",\n \"emscripten/src/settings_internal.js\",\n ] + glob(\n include = [\n \"emscripten/third_party/**\",\n \"emscripten/tools/**\",\n ],\n exclude = [\n \"**/__pycache__/**\",\n ],\n ),\n)\n\nfilegroup(\n name = \"compiler_files\",\n srcs = [\n \"bin/clang.exe\",\n \"bin/clang++.exe\",\n \":emcc_common\",\n \":includes\",\n ],\n)\n\nfilegroup(\n name = \"linker_files\",\n srcs = [\n \"bin/clang.exe\",\n \"bin/llvm-ar.exe\",\n \"bin/llvm-dwarfdump.exe\",\n \"bin/llvm-nm.exe\",\n \"bin/llvm-objcopy.exe\",\n \"bin/wasm-ctor-eval.exe\",\n \"bin/wasm-emscripten-finalize.exe\",\n \"bin/wasm-ld.exe\",\n \"bin/wasm-metadce.exe\",\n \"bin/wasm-opt.exe\",\n \"bin/wasm-split.exe\",\n \"bin/wasm2js.exe\",\n \":emcc_common\",\n ] + glob(\n include = [\n \"emscripten/cache/sysroot/lib/**\",\n \"emscripten/node_modules/**\",\n \"emscripten/src/**\",\n ],\n ),\n)\n\nfilegroup(\n name = \"ar_files\",\n srcs = [\n \"bin/llvm-ar.exe\",\n \"emscripten/emar.py\",\n \"emscripten/emscripten-version.txt\",\n \"emscripten/src/settings.js\",\n \"emscripten/src/settings_internal.js\",\n ] + glob(\n include = [\n \"emscripten/tools/**\",\n ],\n exclude = [\n \"**/__pycache__/**\",\n ],\n ),\n)\n", + "sha256": "bd2094ca9bde5df25020a46ece7f56b622d1d22214fbd12950b01b952dd40084", + "strip_prefix": "install", + "type": "zip", + "url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/win/7033fec38817ec01909b044ea0193ddd5057255c/wasm-binaries.zip" + } + } + }, + "recordedRepoMappingEntries": [ + [ + "bazel_tools", + "rules_cc", + "rules_cc+" + ], + [ + "emsdk+", + "bazel_tools", + "bazel_tools" + ] + ] + } + }, "@@googleapis+//:extensions.bzl%switched_rules": { "general": { "bzlTransitiveDigest": "vG6fuTzXD8MMvHWZEQud0MMH7eoC4GXY0va7VrFFh04=", @@ -894,6 +1322,163 @@ ] ] } + }, + "@@rules_nodejs+//nodejs:extensions.bzl%node": { + "general": { + "bzlTransitiveDigest": "rphcryfYrOY/P3emfTskC/GY5YuHcwMl2B2ncjaM8lY=", + "usagesDigest": "u5PYlUfR7aYByqkL169Tmxkewow8vs35XHttkoUSn0U=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "nodejs_linux_amd64": { + "repoRuleId": "@@rules_nodejs+//nodejs:repositories.bzl%_nodejs_repositories", + "attributes": { + "node_download_auth": {}, + "node_repositories": {}, + "node_urls": [ + "https://nodejs.org/dist/v{version}/{filename}" + ], + "node_version": "20.18.0", + "include_headers": false, + "platform": "linux_amd64" + } + }, + "nodejs_linux_arm64": { + "repoRuleId": "@@rules_nodejs+//nodejs:repositories.bzl%_nodejs_repositories", + "attributes": { + "node_download_auth": {}, + "node_repositories": {}, + "node_urls": [ + "https://nodejs.org/dist/v{version}/{filename}" + ], + "node_version": "20.18.0", + "include_headers": false, + "platform": "linux_arm64" + } + }, + "nodejs_linux_s390x": { + "repoRuleId": "@@rules_nodejs+//nodejs:repositories.bzl%_nodejs_repositories", + "attributes": { + "node_download_auth": {}, + "node_repositories": {}, + "node_urls": [ + "https://nodejs.org/dist/v{version}/{filename}" + ], + "node_version": "20.18.0", + "include_headers": false, + "platform": "linux_s390x" + } + }, + "nodejs_linux_ppc64le": { + "repoRuleId": "@@rules_nodejs+//nodejs:repositories.bzl%_nodejs_repositories", + "attributes": { + "node_download_auth": {}, + "node_repositories": {}, + "node_urls": [ + "https://nodejs.org/dist/v{version}/{filename}" + ], + "node_version": "20.18.0", + "include_headers": false, + "platform": "linux_ppc64le" + } + }, + "nodejs_darwin_amd64": { + "repoRuleId": "@@rules_nodejs+//nodejs:repositories.bzl%_nodejs_repositories", + "attributes": { + "node_download_auth": {}, + "node_repositories": {}, + "node_urls": [ + "https://nodejs.org/dist/v{version}/{filename}" + ], + "node_version": "20.18.0", + "include_headers": false, + "platform": "darwin_amd64" + } + }, + "nodejs_darwin_arm64": { + "repoRuleId": "@@rules_nodejs+//nodejs:repositories.bzl%_nodejs_repositories", + "attributes": { + "node_download_auth": {}, + "node_repositories": {}, + "node_urls": [ + "https://nodejs.org/dist/v{version}/{filename}" + ], + "node_version": "20.18.0", + "include_headers": false, + "platform": "darwin_arm64" + } + }, + "nodejs_windows_amd64": { + "repoRuleId": "@@rules_nodejs+//nodejs:repositories.bzl%_nodejs_repositories", + "attributes": { + "node_download_auth": {}, + "node_repositories": {}, + "node_urls": [ + "https://nodejs.org/dist/v{version}/{filename}" + ], + "node_version": "20.18.0", + "include_headers": false, + "platform": "windows_amd64" + } + }, + "nodejs": { + "repoRuleId": "@@rules_nodejs+//nodejs/private:nodejs_repo_host_os_alias.bzl%nodejs_repo_host_os_alias", + "attributes": { + "user_node_repository_name": "nodejs" + } + }, + "nodejs_host": { + "repoRuleId": "@@rules_nodejs+//nodejs/private:nodejs_repo_host_os_alias.bzl%nodejs_repo_host_os_alias", + "attributes": { + "user_node_repository_name": "nodejs" + } + }, + "nodejs_toolchains": { + "repoRuleId": "@@rules_nodejs+//nodejs/private:nodejs_toolchains_repo.bzl%nodejs_toolchains_repo", + "attributes": { + "user_node_repository_name": "nodejs" + } + } + }, + "recordedRepoMappingEntries": [] + } + }, + "@@rules_python+//python/uv:uv.bzl%uv": { + "general": { + "bzlTransitiveDigest": "Xpqjnjzy6zZ90Es9Wa888ZLHhn7IsNGbph/e6qoxzw8=", + "usagesDigest": "vJ5RHUxAnV24M5swNGiAnkdxMx3Hp/iOLmNANTC5Xc8=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "uv": { + "repoRuleId": "@@rules_python+//python/uv/private:uv_toolchains_repo.bzl%uv_toolchains_repo", + "attributes": { + "toolchain_type": "'@@rules_python+//python/uv:uv_toolchain_type'", + "toolchain_names": [ + "none" + ], + "toolchain_implementations": { + "none": "'@@rules_python+//python:none'" + }, + "toolchain_compatible_with": { + "none": [ + "@platforms//:incompatible" + ] + }, + "toolchain_target_settings": {} + } + } + }, + "recordedRepoMappingEntries": [ + [ + "rules_python+", + "platforms", + "platforms" + ] + ] + } } } } diff --git a/grpc/grpcpp-c/include/protowire.h b/grpc/grpcpp-c/include/protowire.h new file mode 100644 index 000000000..f15c6611b --- /dev/null +++ b/grpc/grpcpp-c/include/protowire.h @@ -0,0 +1,39 @@ +#ifndef PROTOWIRE_H +#define PROTOWIRE_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + + //// WIRE ENCODER //// + + typedef struct pw_encoder pw_encoder_t; + + pw_encoder_t * pw_encoder_new(uint8_t *buf, uint32_t cap); + void pw_encoder_delete(pw_encoder_t *self); + + bool pw_encoder_write_bool(pw_encoder_t *self, int field_no, bool v); + + bool pw_encoder_write_string(pw_encoder_t *self, int field_no, const char *v); + + + //// WIRE DECODER //// + + typedef struct pw_decoder pw_decoder_t; + + pw_decoder_t * pw_decoder_new(uint8_t *buf, uint32_t cap); + void pw_decoder_delete(pw_decoder_t *self); + + uint32_t pw_decoder_read_tag(pw_decoder_t *self); + bool pw_decoder_read_bool(pw_decoder_t *self, bool *v); + bool pw_decoder_read_string(pw_decoder_t *self, void **opaque_string, const char **out); + void pw_decoder_delete_opaque_string(void *opaque_string); + +#ifdef __cplusplus + } +#endif + +#endif //PROTOWIRE_H diff --git a/grpc/grpcpp-c/src/protowire.cpp b/grpc/grpcpp-c/src/protowire.cpp new file mode 100644 index 000000000..0b73ea050 --- /dev/null +++ b/grpc/grpcpp-c/src/protowire.cpp @@ -0,0 +1,95 @@ +// +// Created by Johannes Zottele on 17.07.25. +// + +#include "protowire.h" + +#include "src/google/protobuf/io/zero_copy_stream_impl_lite.h" +#include "src/google/protobuf/io/coded_stream.h" +#include "src/google/protobuf/wire_format_lite.h" + +namespace pb = google::protobuf; + +typedef pb::internal::WireFormatLite WireFormatLite; + +struct pw_encoder { + pb::io::ArrayOutputStream aos; + pb::io::CodedOutputStream cos; + + pw_encoder(uint8_t* buf, int size) + : aos(buf, size), + cos(&aos) {} + +}; + +struct pw_decoder { + pb::io::ArrayInputStream ais; + pb::io::CodedInputStream cis; + + pw_decoder(uint8_t* buf, int size) + : ais(buf, size), + cis(&ais) {} +}; + +extern "C" { + + pw_encoder_t * pw_encoder_new(uint8_t *buf, uint32_t cap) { + return new pw_encoder_t(buf, cap); + } + + void pw_encoder_delete(pw_encoder_t *self) { + delete self; + } + + // check if there was an error + static bool check(pw_encoder_t *self) { + return self->cos.HadError(); + } + + bool pw_encoder_write_bool(pw_encoder_t *self, int field_no, bool v) { + WireFormatLite::WriteBool(field_no, v, &self->cos); + return check(self); + } + + bool pw_encoder_write_string(pw_encoder_t *self, int field_no, const char *v) { + // TODO: This requires a copy of the string. + // We could write the raw buffer manually to avoid this. + const std::string str(v); + WireFormatLite::WriteString(field_no, str, &self->cos); + return check(self); + } + + + pw_decoder_t *pw_decoder_new(uint8_t *buf, uint32_t cap) { + return new pw_decoder_t(buf, cap); + } + + void pw_decoder_delete(pw_decoder_t *self) { + delete self; + } + + uint32_t pw_decoder_read_tag(pw_decoder_t *self) { + return self->cis.ReadTag(); + } + + bool pw_decoder_read_bool(pw_decoder_t *self, bool *v) { + return WireFormatLite::ReadPrimitive(&self->cis, v); + } + + bool pw_decoder_read_string(pw_decoder_t *self, void **opaque_string, const char **out) { + // create a std::string object and place the pointer to the opaque_string ptr references + // this string object will outlive the functions, the caller is responsible to call + // pw_decoder_delete_opaque_string with the opaque string pointer + auto *str_ptr = new std::string; + *opaque_string = str_ptr; + const auto ok = WireFormatLite::ReadString(&self->cis, str_ptr); + // set the c_str start pointer to the out ptr reference + *out = str_ptr->c_str(); + return ok; + } + + void pw_decoder_delete_opaque_string(void *opaque_string) { + auto *str_ptr = static_cast(opaque_string); + delete str_ptr; + } +} From 03ab1554cf8f79a1bfb7860b7ba9a21aa4367779 Mon Sep 17 00:00:00 2001 From: Johannes Zottele Date: Fri, 18 Jul 2025 17:17:40 +0200 Subject: [PATCH 02/16] grpc-native: Add primitive value encoding/decoding Signed-off-by: Johannes Zottele --- .../grpc/internal/bridge/GrpcByteBuffer.kt | 16 ++- .../rpc/grpc/internal/bridge/GrpcClient.kt | 17 ++- .../rpc/grpc/internal/bridge/GrpcSlice.kt | 12 +- .../grpc/internal/protowire/WireDecoder.kt | 125 +++++++++++++++--- .../grpc/internal/protowire/WireEncoder.kt | 58 +++++++- .../kotlin/kotlinx/rpc/grpc/BridgeTest.kt | 36 ----- .../kotlinx/rpc/grpc/internal/BridgeTest.kt | 39 ++++++ .../rpc/grpc/{ => internal}/ProtoWireTest.kt | 13 +- grpc/grpcpp-c/BUILD.bazel | 33 ++--- grpc/grpcpp-c/include/protowire.h | 46 ++++++- grpc/grpcpp-c/src/protowire.cpp | 81 ++++++++---- 11 files changed, 343 insertions(+), 133 deletions(-) delete mode 100644 grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/BridgeTest.kt create mode 100644 grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/BridgeTest.kt rename grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/{ => internal}/ProtoWireTest.kt (80%) diff --git a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/bridge/GrpcByteBuffer.kt b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/bridge/GrpcByteBuffer.kt index 7b4c72694..f5df196b2 100644 --- a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/bridge/GrpcByteBuffer.kt +++ b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/bridge/GrpcByteBuffer.kt @@ -6,16 +6,24 @@ package kotlinx.rpc.grpc.internal.bridge import kotlinx.cinterop.* import libgrpcpp_c.* +import kotlin.experimental.ExperimentalNativeApi +import kotlin.native.ref.createCleaner -@OptIn(ExperimentalForeignApi::class) +@OptIn(ExperimentalForeignApi::class, ExperimentalNativeApi::class) internal class GrpcByteBuffer internal constructor( internal val cByteBuffer: CPointer -) : AutoCloseable { +) { constructor(slice: GrpcSlice) : this(memScoped { grpc_raw_byte_buffer_create(slice.cSlice, 1u) ?: error("Failed to create byte buffer") }) + init { + createCleaner(cByteBuffer) { + grpc_byte_buffer_destroy(it) + } + } + fun intoSlice(): GrpcSlice { memScoped { val respSlice = alloc() @@ -24,8 +32,4 @@ internal class GrpcByteBuffer internal constructor( } } - override fun close() { - grpc_byte_buffer_destroy(cByteBuffer) - } - } diff --git a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/bridge/GrpcClient.kt b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/bridge/GrpcClient.kt index 891d81e79..642712dfa 100644 --- a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/bridge/GrpcClient.kt +++ b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/bridge/GrpcClient.kt @@ -9,12 +9,20 @@ import kotlinx.coroutines.suspendCancellableCoroutine import libgrpcpp_c.* import kotlin.coroutines.resume import kotlin.coroutines.resumeWithException +import kotlin.experimental.ExperimentalNativeApi +import kotlin.native.ref.createCleaner -@OptIn(ExperimentalForeignApi::class) -internal class GrpcClient(target: String) : AutoCloseable { +@OptIn(ExperimentalForeignApi::class, ExperimentalNativeApi::class) +internal class GrpcClient(target: String) { private var clientPtr: CPointer = grpc_client_create_insecure(target) ?: error("Failed to create client") + init { + createCleaner(clientPtr) { + grpc_client_delete(it) + } + } + fun callUnaryBlocking(method: String, req: GrpcSlice): GrpcSlice { memScoped { val result = alloc() @@ -62,9 +70,4 @@ internal class GrpcClient(target: String) : AutoCloseable { cbCtxStable.dispose() }) } - - override fun close() { - grpc_client_delete(clientPtr) - } - } diff --git a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/bridge/GrpcSlice.kt b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/bridge/GrpcSlice.kt index fa8738f0f..70ab9a515 100644 --- a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/bridge/GrpcSlice.kt +++ b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/bridge/GrpcSlice.kt @@ -11,9 +11,11 @@ import kotlinx.cinterop.usePinned import libgrpcpp_c.grpc_slice import libgrpcpp_c.grpc_slice_from_copied_buffer import libgrpcpp_c.grpc_slice_unref +import kotlin.experimental.ExperimentalNativeApi +import kotlin.native.ref.createCleaner -@OptIn(ExperimentalForeignApi::class) -internal class GrpcSlice internal constructor(internal val cSlice: CValue) : AutoCloseable { +@OptIn(ExperimentalForeignApi::class, ExperimentalNativeApi::class) +internal class GrpcSlice internal constructor(internal val cSlice: CValue) { constructor(buffer: ByteArray) : this( buffer.usePinned { pinned -> @@ -21,7 +23,9 @@ internal class GrpcSlice internal constructor(internal val cSlice: CValue() - if (pw_decoder_read_bool(raw, bool.ptr)) { - return bool.value + val value = alloc() + if (pw_decoder_read_bool(raw, value.ptr)) { + return value.value } return null } - fun readString(): String? = memScoped { - // allocate an opaque pointer that holds a reference to the allocated std::string. - // by keeping std::string object alive, we avoid the need to copy the C++ string to an allocated C string. - val opaqueStr = alloc() - val cCharPtr = alloc>() - val ok = pw_decoder_read_string(raw, opaqueStr.ptr, cCharPtr.ptr) - var result: String? = null - if (ok) result = cCharPtr.value?.toKString() - // after copying the string to a Kotlin string, we must delete the allocated C++ string (includes the C string) - pw_decoder_delete_opaque_string(opaqueStr.value) - return result; + fun readInt32(): Int? = memScoped { + val value = alloc() + if (pw_decoder_read_int32(raw, value.ptr)) { + return value.value + } + return null + } + + fun readInt64(): Long? = memScoped { + val value = alloc() + if (pw_decoder_read_int64(raw, value.ptr)) { + return value.value + } + return null + } + + fun readUInt32(): UInt? = memScoped { + val value = alloc() + if (pw_decoder_read_uint32(raw, value.ptr)) { + return value.value + } + return null + } + + fun readUInt64(): ULong? = memScoped { + val value = alloc() + if (pw_decoder_read_uint64(raw, value.ptr)) { + return value.value + } + return null + } + + fun readSInt32(): Int? = memScoped { + val value = alloc() + if (pw_decoder_read_sint32(raw, value.ptr)) { + return value.value + } + return null + } + fun readSInt64(): Long? = memScoped { + val value = alloc() + if (pw_decoder_read_sint64(raw, value.ptr)) { + return value.value + } + return null + } + + fun readFixed32(): UInt? = memScoped { + val value = alloc() + if (pw_decoder_read_fixed32(raw, value.ptr)) { + return value.value + } + return null + } + + fun readFixed64(): ULong? = memScoped { + val value = alloc() + if (pw_decoder_read_fixed64(raw, value.ptr)) { + return value.value + } + return null + } + + fun readSFixed32(): Int? = memScoped { + val value = alloc() + if (pw_decoder_read_sfixed32(raw, value.ptr)) { + return value.value + } + return null + } + + fun readSFixed64(): Long? = memScoped { + val value = alloc() + if (pw_decoder_read_sfixed64(raw, value.ptr)) { + return value.value + } + return null + } + + fun readEnum(): Int? = memScoped { + val value = alloc() + if (pw_decoder_read_enum(raw, value.ptr)) { + return value.value + } + return null + } + + fun readString(): String? = memScoped { + val str = alloc>() + val ok = pw_decoder_read_string(raw, str.ptr) + try { + if (!ok) return null + return pw_string_c_str(str.value)?.toKString() + } finally { + pw_string_delete(str.value) + } } } diff --git a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/protowire/WireEncoder.kt b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/protowire/WireEncoder.kt index 1c54d7d24..53c753949 100644 --- a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/protowire/WireEncoder.kt +++ b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/protowire/WireEncoder.kt @@ -8,10 +8,7 @@ import kotlinx.cinterop.ExperimentalForeignApi import kotlinx.cinterop.Pinned import kotlinx.cinterop.addressOf import kotlinx.cinterop.pin -import libprotowire.pw_encoder_delete -import libprotowire.pw_encoder_new -import libprotowire.pw_encoder_write_bool -import libprotowire.pw_encoder_write_string +import libprotowire.* import kotlin.experimental.ExperimentalNativeApi import kotlin.native.ref.createCleaner @@ -29,12 +26,59 @@ internal class WireEncoder { } } - fun write(field: Int, value: Boolean): Boolean { + fun writeBool(field: Int, value: Boolean): Boolean { return pw_encoder_write_bool(raw, field, value) } - fun write(field: Int, value: String): Boolean { - return pw_encoder_write_string(raw, field, value) + fun writeInt32(fieldNr: Int, value: Int): Boolean { + return pw_encoder_write_int32(raw, fieldNr, value) + } + + fun writeInt64(fieldNr: Int, value: Long): Boolean { + return pw_encoder_write_int64(raw, fieldNr, value) + } + + fun writeUInt32(fieldNr: Int, value: UInt): Boolean { + return pw_encoder_write_uint32(raw, fieldNr, value) + } + + fun writeUInt64(fieldNr: Int, value: ULong): Boolean { + return pw_encoder_write_uint64(raw, fieldNr, value) + } + + fun writeSInt32(fieldNr: Int, value: Int): Boolean { + return pw_encoder_write_sint32(raw, fieldNr, value) + } + + fun writeSInt64(fieldNr: Int, value: Long): Boolean { + return pw_encoder_write_sint64(raw, fieldNr, value) + } + + fun writeFixed32(fieldNr: Int, value: UInt): Boolean { + return pw_encoder_write_fixed32(raw, fieldNr, value) + } + + fun writeFixed64(fieldNr: Int, value: ULong): Boolean { + return pw_encoder_write_fixed64(raw, fieldNr, value) + } + + fun writeSFixed32(fieldNr: Int, value: Int): Boolean { + return pw_encoder_write_sfixed32(raw, fieldNr, value) + } + + fun writeSFixed64(fieldNr: Int, value: Long): Boolean { + return pw_encoder_write_sfixed64(raw, fieldNr, value) + } + + fun writeEnum(fieldNr: Int, value: Int): Boolean { + return pw_encoder_write_enum(raw, fieldNr, value) + } + + fun writeString(field: Int, value: String): Boolean { + val str = pw_string_new(value) ?: error("Failed to create string") + val result = pw_encoder_write_string(raw, field, str) + pw_string_delete(str) + return result; } } \ No newline at end of file diff --git a/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/BridgeTest.kt b/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/BridgeTest.kt deleted file mode 100644 index 3a9a40e9c..000000000 --- a/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/BridgeTest.kt +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2023-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. - */ - -package kotlinx.rpc.grpc - -import kotlinx.coroutines.runBlocking -import kotlinx.rpc.grpc.internal.bridge.GrpcByteBuffer -import kotlinx.rpc.grpc.internal.bridge.GrpcClient -import kotlinx.rpc.grpc.internal.bridge.GrpcSlice -import libgrpcpp_c.pb_decode_greeter_sayhello_response -import kotlin.test.Test - -@OptIn(kotlinx.cinterop.ExperimentalForeignApi::class) -class BridgeTest { - - @Test - fun testBasicUnaryAsyncCall() { - runBlocking { - GrpcClient("localhost:50051").use { client -> - GrpcSlice(byteArrayOf(8, 4)).use { request -> - GrpcByteBuffer(request).use { req_buf -> - client.callUnary("/Greeter/SayHello", req_buf) - .use { result -> - result.intoSlice().use { response -> - val value = pb_decode_greeter_sayhello_response(response.cSlice) - println("Response received: $value") - } - - } - } - } - } - } - } -} diff --git a/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/BridgeTest.kt b/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/BridgeTest.kt new file mode 100644 index 000000000..5c3b8b97a --- /dev/null +++ b/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/BridgeTest.kt @@ -0,0 +1,39 @@ +/* + * Copyright 2023-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.rpc.grpc.internal + +import kotlinx.cinterop.ExperimentalForeignApi +import kotlinx.coroutines.runBlocking +import kotlinx.rpc.grpc.internal.bridge.GrpcByteBuffer +import kotlinx.rpc.grpc.internal.bridge.GrpcClient +import kotlinx.rpc.grpc.internal.bridge.GrpcSlice +import libgrpcpp_c.pb_decode_greeter_sayhello_response +import kotlin.native.runtime.GC +import kotlin.native.runtime.NativeRuntimeApi +import kotlin.test.Test +import kotlin.test.fail + +@OptIn(ExperimentalForeignApi::class) +class BridgeTest { + + @OptIn(NativeRuntimeApi::class) + @Test + fun testBasicUnaryAsyncCall() = runBlocking { + try { + val client = GrpcClient("localhost:50051") + val request = GrpcSlice(byteArrayOf(8, 4)) + val reqBuf = GrpcByteBuffer(request) + val result = client.callUnary("/Greeter/SayHello", reqBuf) + val response = result.intoSlice() + val value = pb_decode_greeter_sayhello_response(response.cSlice) + println("Response received: $value") + } catch (e: Exception) { + // trigger GC collection, otherwise there will be a leak + GC.collect() + fail("Got an exception: ${e.message}", e) + } + } + +} diff --git a/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/ProtoWireTest.kt b/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/ProtoWireTest.kt similarity index 80% rename from grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/ProtoWireTest.kt rename to grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/ProtoWireTest.kt index 677088633..41df1227f 100644 --- a/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/ProtoWireTest.kt +++ b/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/ProtoWireTest.kt @@ -2,25 +2,26 @@ * Copyright 2023-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. */ -package kotlinx.rpc.grpc +package kotlinx.rpc.grpc.internal import kotlinx.cinterop.ExperimentalForeignApi import kotlinx.rpc.grpc.internal.protowire.WireDecoder import kotlinx.rpc.grpc.internal.protowire.WireEncoder import kotlinx.rpc.grpc.internal.protowire.WireType +import kotlin.experimental.ExperimentalNativeApi import kotlin.test.Test @OptIn(ExperimentalForeignApi::class) class ProtoWireTest { + @OptIn(ExperimentalNativeApi::class) @Test - fun testEncodeDecodeBool() { - + fun testEncodeDecodeBool() { val fieldNr = 3; val encoder = WireEncoder() - encoder.write(fieldNr, true) - encoder.write(4, "Hello Test") + encoder.writeBool(fieldNr, true) + encoder.writeString(4, "Hello Test") val encodedBuffer = encoder.buffer.get() val decoder = WireDecoder(encodedBuffer) @@ -38,7 +39,5 @@ class ProtoWireTest { val str = decoder.readString()!! check(str == "Hello Test") - - } } diff --git a/grpc/grpcpp-c/BUILD.bazel b/grpc/grpcpp-c/BUILD.bazel index 0cce2f26a..da41f770c 100644 --- a/grpc/grpcpp-c/BUILD.bazel +++ b/grpc/grpcpp-c/BUILD.bazel @@ -1,28 +1,24 @@ load("@emsdk//emscripten_toolchain:wasm_rules.bzl", "wasm_cc_binary") load("@rules_cc//cc:defs.bzl", "cc_library") -cc_library( - name = "grpcpp_c", - srcs = ["src/grpcpp_c.cpp"], - hdrs = glob(["include/**/*.h"]), - copts = ["-std=c++20"], - includes = ["include"], - visibility = ["//visibility:public"], +cc_static_library( + name = "grpcpp_c_static", deps = [ - "@com_github_grpc_grpc//:grpc++", - "@com_google_protobuf//:protobuf", + ":grpcpp_c", ], ) cc_library( - name = "protowire", - srcs = ["src/protowire.cpp"], - hdrs = glob(["include/*.h"]), + name = "grpcpp_c", + srcs = ["src/grpcpp_c.cpp"], + hdrs = glob(["include/**/*.h"]), copts = ["-std=c++20"], includes = ["include"], visibility = ["//visibility:public"], deps = [ - "@com_google_protobuf//:protobuf_lite", + "@com_github_grpc_grpc//:channelz", + "@com_github_grpc_grpc//:generic_stub", + "@com_github_grpc_grpc//:grpc_credentials_util", ], ) @@ -33,9 +29,14 @@ cc_static_library( ], ) -cc_static_library( - name = "grpcpp_c_static", +cc_library( + name = "protowire", + srcs = ["src/protowire.cpp"], + hdrs = glob(["include/*.h"]), + copts = ["-std=c++20"], + includes = ["include"], + visibility = ["//visibility:public"], deps = [ - "grpcpp_c", + "@com_google_protobuf//:protobuf_lite", ], ) diff --git a/grpc/grpcpp-c/include/protowire.h b/grpc/grpcpp-c/include/protowire.h index f15c6611b..6692269e8 100644 --- a/grpc/grpcpp-c/include/protowire.h +++ b/grpc/grpcpp-c/include/protowire.h @@ -8,6 +8,13 @@ extern "C" { #endif + typedef struct pw_string pw_string_t; + + pw_string_t * pw_string_new(const char *str); + void pw_string_delete(pw_string_t *self); + const char * pw_string_c_str(pw_string_t *self); + + //// WIRE ENCODER //// typedef struct pw_encoder pw_encoder_t; @@ -15,9 +22,23 @@ extern "C" { pw_encoder_t * pw_encoder_new(uint8_t *buf, uint32_t cap); void pw_encoder_delete(pw_encoder_t *self); - bool pw_encoder_write_bool(pw_encoder_t *self, int field_no, bool v); + bool pw_encoder_write_bool(pw_encoder_t *self, int field_no, bool value); + bool pw_encoder_write_int32(pw_encoder_t *self, int field_no, int32_t value); + bool pw_encoder_write_int64(pw_encoder_t *self, int field_no, int64_t value); + bool pw_encoder_write_uint32(pw_encoder_t *self, int field_no, uint32_t value); + bool pw_encoder_write_uint64(pw_encoder_t *self, int field_no, uint64_t value); + bool pw_encoder_write_sint32(pw_encoder_t *self, int field_no, int32_t value); + bool pw_encoder_write_sint64(pw_encoder_t *self, int field_no, int64_t value); + bool pw_encoder_write_fixed32(pw_encoder_t *self, int field_no, uint32_t value); + bool pw_encoder_write_fixed64(pw_encoder_t *self, int field_no, uint64_t value); + bool pw_encoder_write_sfixed32(pw_encoder_t *self, int field_no, int32_t value); + bool pw_encoder_write_sfixed64(pw_encoder_t *self, int field_no, int64_t value); + bool pw_encoder_write_float(pw_encoder_t *self, int field_no, float value); + bool pw_encoder_write_double(pw_encoder_t *self, int field_no, double value); + bool pw_encoder_write_enum(pw_encoder_t *self, int field_no, int value); + bool pw_encoder_write_string(pw_encoder_t *self, int field_no, pw_string_t *value); + bool pw_encoder_write_bytes(pw_encoder_t *self, int field_no, pw_string_t *value); - bool pw_encoder_write_string(pw_encoder_t *self, int field_no, const char *v); //// WIRE DECODER //// @@ -28,9 +49,24 @@ extern "C" { void pw_decoder_delete(pw_decoder_t *self); uint32_t pw_decoder_read_tag(pw_decoder_t *self); - bool pw_decoder_read_bool(pw_decoder_t *self, bool *v); - bool pw_decoder_read_string(pw_decoder_t *self, void **opaque_string, const char **out); - void pw_decoder_delete_opaque_string(void *opaque_string); + + /** Read primitive **/ + + bool pw_decoder_read_bool(pw_decoder_t *self, bool *value); + bool pw_decoder_read_int32(pw_decoder_t *self, int32_t *value); + bool pw_decoder_read_int64(pw_decoder_t *self, int64_t *value); + bool pw_decoder_read_uint32(pw_decoder_t *self, uint32_t *value); + bool pw_decoder_read_uint64(pw_decoder_t *self, uint64_t *value); + bool pw_decoder_read_sint32(pw_decoder_t *self, int32_t *value); + bool pw_decoder_read_sint64(pw_decoder_t *self, int64_t *value); + bool pw_decoder_read_fixed32(pw_decoder_t *self, uint32_t *value); + bool pw_decoder_read_fixed64(pw_decoder_t *self, uint64_t *value); + bool pw_decoder_read_sfixed32(pw_decoder_t *self, int32_t *value); + bool pw_decoder_read_sfixed64(pw_decoder_t *self, int64_t *value); + bool pw_decoder_read_float(pw_decoder_t *self, float *value); + bool pw_decoder_read_double(pw_decoder_t *self, double *value); + bool pw_decoder_read_enum(pw_decoder_t *self, int *value); + bool pw_decoder_read_string(pw_decoder_t *self, pw_string_t **opaque_string); #ifdef __cplusplus } diff --git a/grpc/grpcpp-c/src/protowire.cpp b/grpc/grpcpp-c/src/protowire.cpp index 0b73ea050..652352fac 100644 --- a/grpc/grpcpp-c/src/protowire.cpp +++ b/grpc/grpcpp-c/src/protowire.cpp @@ -12,6 +12,10 @@ namespace pb = google::protobuf; typedef pb::internal::WireFormatLite WireFormatLite; +struct pw_string { + std::string str; +}; + struct pw_encoder { pb::io::ArrayOutputStream aos; pb::io::CodedOutputStream cos; @@ -33,6 +37,16 @@ struct pw_decoder { extern "C" { + pw_string_t *pw_string_new(const char *str) { + return new pw_string_t{str }; + } + void pw_string_delete(pw_string_t *self) { + delete self; + } + const char *pw_string_c_str(pw_string_t *self) { + return self->str.c_str(); + } + pw_encoder_t * pw_encoder_new(uint8_t *buf, uint32_t cap) { return new pw_encoder_t(buf, cap); } @@ -46,16 +60,31 @@ extern "C" { return self->cos.HadError(); } - bool pw_encoder_write_bool(pw_encoder_t *self, int field_no, bool v) { - WireFormatLite::WriteBool(field_no, v, &self->cos); - return check(self); +#define WRITE_FIELD_FUNC( funcSuffix, wireTy, cTy) \ + bool pw_encoder_write_##funcSuffix(pw_encoder_t *self, int field_no, cTy value) { \ + WireFormatLite::Write##wireTy(field_no, value, &self->cos); \ + return check(self); \ } - bool pw_encoder_write_string(pw_encoder_t *self, int field_no, const char *v) { - // TODO: This requires a copy of the string. - // We could write the raw buffer manually to avoid this. - const std::string str(v); - WireFormatLite::WriteString(field_no, str, &self->cos); + WRITE_FIELD_FUNC( bool, Bool, bool) + WRITE_FIELD_FUNC( int32, Int32, int32_t) + WRITE_FIELD_FUNC( int64, Int64, int64_t) + WRITE_FIELD_FUNC( uint32, UInt32, uint32_t) + WRITE_FIELD_FUNC( uint64, UInt64, uint64_t) + WRITE_FIELD_FUNC( sint32, SInt32, int32_t) + WRITE_FIELD_FUNC( sint64, SInt64, int64_t) + WRITE_FIELD_FUNC( fixed32, Fixed32, uint32_t) + WRITE_FIELD_FUNC( fixed64, Fixed64, uint64_t) + WRITE_FIELD_FUNC( sfixed32, SFixed32, int32_t) + WRITE_FIELD_FUNC( sfixed64, SFixed64, int64_t) + WRITE_FIELD_FUNC( enum, Enum, int) + + bool pw_encoder_write_string(pw_encoder_t *self, int field_no, pw_string_t *value) { + WireFormatLite::WriteString(field_no, value->str, &self->cos); + return check(self); + } + bool pw_encoder_write_bytes(pw_encoder_t *self, int field_no, pw_string_t *value) { + WireFormatLite::WriteBytes(field_no, value->str, &self->cos); return check(self); } @@ -72,24 +101,26 @@ extern "C" { return self->cis.ReadTag(); } - bool pw_decoder_read_bool(pw_decoder_t *self, bool *v) { - return WireFormatLite::ReadPrimitive(&self->cis, v); - } - - bool pw_decoder_read_string(pw_decoder_t *self, void **opaque_string, const char **out) { - // create a std::string object and place the pointer to the opaque_string ptr references - // this string object will outlive the functions, the caller is responsible to call - // pw_decoder_delete_opaque_string with the opaque string pointer - auto *str_ptr = new std::string; - *opaque_string = str_ptr; - const auto ok = WireFormatLite::ReadString(&self->cis, str_ptr); - // set the c_str start pointer to the out ptr reference - *out = str_ptr->c_str(); - return ok; +#define READ_VAL_FUNC( funcSuffix, wireTy, cTy) \ + bool pw_decoder_read_##funcSuffix(pw_decoder_t *self, cTy *value_ref) { \ + return WireFormatLite::ReadPrimitive(&self->cis, value_ref); \ } - void pw_decoder_delete_opaque_string(void *opaque_string) { - auto *str_ptr = static_cast(opaque_string); - delete str_ptr; + READ_VAL_FUNC( bool, BOOL, bool) + READ_VAL_FUNC( int32, INT32, int32_t) + READ_VAL_FUNC( int64, INT64, int64_t) + READ_VAL_FUNC( uint32, UINT32, uint32_t) + READ_VAL_FUNC( uint64, UINT64, uint64_t) + READ_VAL_FUNC( sint32, SINT32, int32_t) + READ_VAL_FUNC( sint64, SINT64, int64_t) + READ_VAL_FUNC( fixed32, FIXED32, uint32_t) + READ_VAL_FUNC( fixed64, FIXED64, uint64_t) + READ_VAL_FUNC( sfixed32, SFIXED32, int32_t) + READ_VAL_FUNC( sfixed64, SFIXED64, int64_t) + READ_VAL_FUNC( enum, ENUM, int) + + bool pw_decoder_read_string(pw_decoder_t *self, pw_string_t **string_ref) { + *string_ref = new pw_string_t; + return WireFormatLite::ReadString(&self->cis, &(*string_ref)->str); } } From b889fc0f74e6996c6d7e469d7625c1107c8021d7 Mon Sep 17 00:00:00 2001 From: Johannes Zottele Date: Mon, 21 Jul 2025 16:09:46 +0200 Subject: [PATCH 03/16] grpc-native: Add common interface for encoder/decoder Signed-off-by: Johannes Zottele --- grpc/grpc-core/build.gradle.kts | 2 ++ .../kotlin/kotlinx/rpc/grpc/internal}/KTag.kt | 4 +--- .../kotlinx/rpc/grpc/internal/WireDecoder.kt | 22 +++++++++++++++++++ .../kotlinx/rpc/grpc/internal/WireEncoder.kt | 22 +++++++++++++++++++ .../kotlinx/rpc/grpc/internal/WireDecoder.kt | 21 ++++++++++++++++++ .../kotlinx/rpc/grpc/internal/WireEncoder.kt | 21 ++++++++++++++++++ 6 files changed, 89 insertions(+), 3 deletions(-) rename grpc/grpc-core/src/{nativeMain/kotlin/kotlinx/rpc/grpc/internal/protowire => commonMain/kotlin/kotlinx/rpc/grpc/internal}/KTag.kt (95%) create mode 100644 grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.kt create mode 100644 grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.kt create mode 100644 grpc/grpc-core/src/jvmMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.kt create mode 100644 grpc/grpc-core/src/jvmMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.kt diff --git a/grpc/grpc-core/build.gradle.kts b/grpc/grpc-core/build.gradle.kts index 0444a88b9..372b06091 100644 --- a/grpc/grpc-core/build.gradle.kts +++ b/grpc/grpc-core/build.gradle.kts @@ -14,6 +14,8 @@ plugins { alias(libs.plugins.kotlinx.rpc) } + + kotlin { compilerOptions { freeCompilerArgs.add("-Xexpect-actual-classes") diff --git a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/protowire/KTag.kt b/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/KTag.kt similarity index 95% rename from grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/protowire/KTag.kt rename to grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/KTag.kt index e6f789651..c77fdfd50 100644 --- a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/protowire/KTag.kt +++ b/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/KTag.kt @@ -2,7 +2,7 @@ * Copyright 2023-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. */ -package kotlinx.rpc.grpc.internal.protowire +package kotlinx.rpc.grpc.internal internal enum class WireType { VARINT, // 0 @@ -14,8 +14,6 @@ internal enum class WireType { } internal data class KTag(val fieldNr: Int, val wireType: WireType) { - - companion object { // Number of bits in a tag which identify the wire type. const val K_TAG_TYPE_BITS: UInt = 3u; diff --git a/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.kt b/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.kt new file mode 100644 index 000000000..f48064269 --- /dev/null +++ b/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.kt @@ -0,0 +1,22 @@ +/* + * Copyright 2023-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.rpc.grpc.internal + +internal interface WireDecoder { + fun readTag(): KTag? + fun readBool(): Boolean? + fun readInt32(): Int? + fun readInt64(): Long? + fun readUInt32(): UInt? + fun readUInt64(): ULong? + fun readSInt32(): Int? + fun readSInt64(): Long? + fun readFixed32(): UInt? + fun readFixed64(): ULong? + fun readSFixed32(): Int? + fun readSFixed64(): Long? + fun readEnum(): Int? + fun readString(): String? +} \ No newline at end of file diff --git a/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.kt b/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.kt new file mode 100644 index 000000000..07bdd4b07 --- /dev/null +++ b/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.kt @@ -0,0 +1,22 @@ +/* + * Copyright 2023-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.rpc.grpc.internal + +internal interface WireEncoder { + fun readTag(): KTag? + fun readBool(): Boolean? + fun readInt32(): Int? + fun readInt64(): Long? + fun readUInt32(): UInt? + fun readUInt64(): ULong? + fun readSInt32(): Int? + fun readSInt64(): Long? + fun readFixed32(): UInt? + fun readFixed64(): ULong? + fun readSFixed32(): Int? + fun readSFixed64(): Long? + fun readEnum(): Int? + fun readString(): String? +} \ No newline at end of file diff --git a/grpc/grpc-core/src/jvmMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.kt b/grpc/grpc-core/src/jvmMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.kt new file mode 100644 index 000000000..63aa45c19 --- /dev/null +++ b/grpc/grpc-core/src/jvmMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.kt @@ -0,0 +1,21 @@ +/* + * Copyright 2023-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.rpc.grpc.internal + +import com.google.protobuf.CodedInputStream +import com.google.protobuf.CodedOutputStream + +@OptIn(ExperimentalUnsignedTypes::class) +internal class WireDecoder(val buffer: ByteArray) { + private val cos = CodedInputStream.newInstance(buffer) + + fun readTag(): Int { + return cos.readTag() + } + + fun readInt32(): Int { + return cos.readRawVarint32() + } +} \ No newline at end of file diff --git a/grpc/grpc-core/src/jvmMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.kt b/grpc/grpc-core/src/jvmMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.kt new file mode 100644 index 000000000..6ceb1051d --- /dev/null +++ b/grpc/grpc-core/src/jvmMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.kt @@ -0,0 +1,21 @@ +/* + * Copyright 2023-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.rpc.grpc.internal + +internal interface WireEncoder(sink: Sink) { + fun writeBool(field: Int, value: Boolean): Boolean + fun writeInt32(fieldNr: Int, value: Int): Boolean + fun writeInt64(fieldNr: Int, value: Long): Boolean + fun writeUInt32(fieldNr: Int, value: UInt): Boolean + fun writeUInt64(fieldNr: Int, value: ULong): Boolean + fun writeSInt32(fieldNr: Int, value: Int): Boolean + fun writeSInt64(fieldNr: Int, value: Long): Boolean + fun writeFixed32(fieldNr: Int, value: UInt): Boolean + fun writeFixed64(fieldNr: Int, value: ULong): Boolean + fun writeSFixed32(fieldNr: Int, value: Int): Boolean + fun writeSFixed64(fieldNr: Int, value: Long): Boolean + fun writeEnum(fieldNr: Int, value: Int): Boolean + fun writeString(field: Int, value: String): Boolean +} \ No newline at end of file From e247e8c27241f2c5bd722826871369097d3263a0 Mon Sep 17 00:00:00 2001 From: Johannes Zottele Date: Tue, 22 Jul 2025 17:55:48 +0200 Subject: [PATCH 04/16] grpc-native: Add the first common / native interface bridge for encoder and decoder with buffers Signed-off-by: Johannes Zottele --- grpc/grpc-core/build.gradle.kts | 1 + .../kotlinx/rpc/grpc/internal/WireDecoder.kt | 24 ++- .../kotlinx/rpc/grpc/internal/WireEncoder.kt | 43 +++-- .../rpc/grpc/internal/WireDecoder.js.kt | 12 ++ .../rpc/grpc/internal/WireEncoder.js.kt | 11 ++ .../rpc/grpc/internal/WireDecoder.jvm.kt | 5 + .../rpc/grpc/internal/WireDecoder.jvm.kt | 12 ++ .../rpc/grpc/internal/WireEncoder.jvm.kt | 11 ++ .../kotlinx/rpc/grpc/internal/WireEncoder.kt | 21 --- .../rpc/grpc/internal/WireDecoder.native.kt | 171 +++++++++++++++++ .../rpc/grpc/internal/WireEncoder.native.kt | 108 +++++++++++ .../rpc/grpc/internal/ZeroCopyInputSource.kt | 176 ++++++++++++++++++ .../grpc/internal/bufferUnsafeExtensions.kt | 30 +++ .../grpc/internal/protowire/WireDecoder.kt | 142 -------------- .../grpc/internal/protowire/WireEncoder.kt | 84 --------- .../rpc/grpc/internal/ProtoWireTest.kt | 14 +- .../grpc/internal/ZeroCopyInputSourceTest.kt | 82 ++++++++ .../rpc/grpc/internal/WireDecoder.wasmJs.kt | 12 ++ .../rpc/grpc/internal/WireEncoder.wasmJs.kt | 11 ++ grpc/grpcpp-c/BUILD.bazel | 11 ++ grpc/grpcpp-c/include/protowire.h | 13 +- grpc/grpcpp-c/src/protowire.cpp | 83 +++++++-- versions-root/libs.versions.toml | 2 + 23 files changed, 794 insertions(+), 285 deletions(-) create mode 100644 grpc/grpc-core/src/jsMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.js.kt create mode 100644 grpc/grpc-core/src/jsMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.js.kt create mode 100644 grpc/grpc-core/src/jvmMain/java/kotlinx/rpc/grpc/internal/WireDecoder.jvm.kt create mode 100644 grpc/grpc-core/src/jvmMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.jvm.kt create mode 100644 grpc/grpc-core/src/jvmMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.jvm.kt delete mode 100644 grpc/grpc-core/src/jvmMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.kt create mode 100644 grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.native.kt create mode 100644 grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.native.kt create mode 100644 grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/ZeroCopyInputSource.kt create mode 100644 grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/bufferUnsafeExtensions.kt delete mode 100644 grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/protowire/WireDecoder.kt delete mode 100644 grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/protowire/WireEncoder.kt create mode 100644 grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/ZeroCopyInputSourceTest.kt create mode 100644 grpc/grpc-core/src/wasmJsMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.wasmJs.kt create mode 100644 grpc/grpc-core/src/wasmJsMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.wasmJs.kt diff --git a/grpc/grpc-core/build.gradle.kts b/grpc/grpc-core/build.gradle.kts index 372b06091..243b912e2 100644 --- a/grpc/grpc-core/build.gradle.kts +++ b/grpc/grpc-core/build.gradle.kts @@ -29,6 +29,7 @@ kotlin { api(libs.coroutines.core) implementation(libs.atomicfu) + implementation(libs.kotlinx.io.core) } } diff --git a/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.kt b/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.kt index f48064269..e3d7f6cbb 100644 --- a/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.kt +++ b/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.kt @@ -4,7 +4,15 @@ package kotlinx.rpc.grpc.internal -internal interface WireDecoder { +import kotlinx.io.Buffer + +/** + * A platform-specific decoder for wire format data. + * + * If one `read*()` method returns `null`, decoding the data failed and no further + * decoding can be done. + */ +internal interface WireDecoder: AutoCloseable { fun readTag(): KTag? fun readBool(): Boolean? fun readInt32(): Int? @@ -19,4 +27,16 @@ internal interface WireDecoder { fun readSFixed64(): Long? fun readEnum(): Int? fun readString(): String? -} \ No newline at end of file +} + +/** + * Creates a platform-specific [WireDecoder]. + * + * This constructor takes a [Buffer] instead of a [kotlinx.io.Source] because + * the native implementation (`WireDecoderNative`) depends on [Buffer]'s internal structure. + * + * NOTE: Do not use the [source] buffer while the [WireDecoder] is still open. + * + * @param source The buffer containing the encoded wire-format data. + */ +internal expect fun WireDecoder(source: Buffer): WireDecoder \ No newline at end of file diff --git a/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.kt b/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.kt index 07bdd4b07..f13aa4f86 100644 --- a/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.kt +++ b/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.kt @@ -4,19 +4,32 @@ package kotlinx.rpc.grpc.internal +import kotlinx.io.Sink + +/** + * A platform-specific class that encodes values into protobuf's wire format. + * + * If one `write*()` method returns false, the encoding of the value failed + * and no further encodings can be performed on this [WireEncoder]. + * + * [flush] must be called to ensure that all data is written to the [Sink]. + */ internal interface WireEncoder { - fun readTag(): KTag? - fun readBool(): Boolean? - fun readInt32(): Int? - fun readInt64(): Long? - fun readUInt32(): UInt? - fun readUInt64(): ULong? - fun readSInt32(): Int? - fun readSInt64(): Long? - fun readFixed32(): UInt? - fun readFixed64(): ULong? - fun readSFixed32(): Int? - fun readSFixed64(): Long? - fun readEnum(): Int? - fun readString(): String? -} \ No newline at end of file + fun writeBool(field: Int, value: Boolean): Boolean + fun writeInt32(fieldNr: Int, value: Int): Boolean + fun writeInt64(fieldNr: Int, value: Long): Boolean + fun writeUInt32(fieldNr: Int, value: UInt): Boolean + fun writeUInt64(fieldNr: Int, value: ULong): Boolean + fun writeSInt32(fieldNr: Int, value: Int): Boolean + fun writeSInt64(fieldNr: Int, value: Long): Boolean + fun writeFixed32(fieldNr: Int, value: UInt): Boolean + fun writeFixed64(fieldNr: Int, value: ULong): Boolean + fun writeSFixed32(fieldNr: Int, value: Int): Boolean + fun writeSFixed64(fieldNr: Int, value: Long): Boolean + fun writeEnum(fieldNr: Int, value: Int): Boolean + fun writeString(fieldNr: Int, value: String): Boolean + fun flush() +} + + +internal expect fun WireEncoder(sink: Sink): WireEncoder \ No newline at end of file diff --git a/grpc/grpc-core/src/jsMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.js.kt b/grpc/grpc-core/src/jsMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.js.kt new file mode 100644 index 000000000..f56885157 --- /dev/null +++ b/grpc/grpc-core/src/jsMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.js.kt @@ -0,0 +1,12 @@ +/* + * Copyright 2023-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.rpc.grpc.internal + +import kotlinx.io.Buffer +import kotlinx.io.Source + +internal actual fun WireDecoder(source: Buffer): WireDecoder { + TODO("Not yet implemented") +} \ No newline at end of file diff --git a/grpc/grpc-core/src/jsMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.js.kt b/grpc/grpc-core/src/jsMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.js.kt new file mode 100644 index 000000000..00c0b3246 --- /dev/null +++ b/grpc/grpc-core/src/jsMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.js.kt @@ -0,0 +1,11 @@ +/* + * Copyright 2023-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.rpc.grpc.internal + +import kotlinx.io.Sink + +internal actual fun WireEncoder(sink: Sink): WireEncoder { + TODO("Not yet implemented") +} \ No newline at end of file diff --git a/grpc/grpc-core/src/jvmMain/java/kotlinx/rpc/grpc/internal/WireDecoder.jvm.kt b/grpc/grpc-core/src/jvmMain/java/kotlinx/rpc/grpc/internal/WireDecoder.jvm.kt new file mode 100644 index 000000000..08a22f14b --- /dev/null +++ b/grpc/grpc-core/src/jvmMain/java/kotlinx/rpc/grpc/internal/WireDecoder.jvm.kt @@ -0,0 +1,5 @@ +/* + * Copyright 2023-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.rpc.grpc.internal \ No newline at end of file diff --git a/grpc/grpc-core/src/jvmMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.jvm.kt b/grpc/grpc-core/src/jvmMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.jvm.kt new file mode 100644 index 000000000..f56885157 --- /dev/null +++ b/grpc/grpc-core/src/jvmMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.jvm.kt @@ -0,0 +1,12 @@ +/* + * Copyright 2023-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.rpc.grpc.internal + +import kotlinx.io.Buffer +import kotlinx.io.Source + +internal actual fun WireDecoder(source: Buffer): WireDecoder { + TODO("Not yet implemented") +} \ No newline at end of file diff --git a/grpc/grpc-core/src/jvmMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.jvm.kt b/grpc/grpc-core/src/jvmMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.jvm.kt new file mode 100644 index 000000000..00c0b3246 --- /dev/null +++ b/grpc/grpc-core/src/jvmMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.jvm.kt @@ -0,0 +1,11 @@ +/* + * Copyright 2023-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.rpc.grpc.internal + +import kotlinx.io.Sink + +internal actual fun WireEncoder(sink: Sink): WireEncoder { + TODO("Not yet implemented") +} \ No newline at end of file diff --git a/grpc/grpc-core/src/jvmMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.kt b/grpc/grpc-core/src/jvmMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.kt deleted file mode 100644 index 6ceb1051d..000000000 --- a/grpc/grpc-core/src/jvmMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.kt +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright 2023-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. - */ - -package kotlinx.rpc.grpc.internal - -internal interface WireEncoder(sink: Sink) { - fun writeBool(field: Int, value: Boolean): Boolean - fun writeInt32(fieldNr: Int, value: Int): Boolean - fun writeInt64(fieldNr: Int, value: Long): Boolean - fun writeUInt32(fieldNr: Int, value: UInt): Boolean - fun writeUInt64(fieldNr: Int, value: ULong): Boolean - fun writeSInt32(fieldNr: Int, value: Int): Boolean - fun writeSInt64(fieldNr: Int, value: Long): Boolean - fun writeFixed32(fieldNr: Int, value: UInt): Boolean - fun writeFixed64(fieldNr: Int, value: ULong): Boolean - fun writeSFixed32(fieldNr: Int, value: Int): Boolean - fun writeSFixed64(fieldNr: Int, value: Long): Boolean - fun writeEnum(fieldNr: Int, value: Int): Boolean - fun writeString(field: Int, value: String): Boolean -} \ No newline at end of file diff --git a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.native.kt b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.native.kt new file mode 100644 index 000000000..3e6b2428f --- /dev/null +++ b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.native.kt @@ -0,0 +1,171 @@ +/* + * Copyright 2023-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.rpc.grpc.internal + +import kotlinx.cinterop.* +import kotlinx.io.Buffer +import libprotowire.* +import kotlin.experimental.ExperimentalNativeApi + +@OptIn(ExperimentalForeignApi::class, ExperimentalNativeApi::class) +internal class WireDecoderNative(private val source: Buffer): WireDecoder { + + // wraps the source in a class that allows to pass data from the source buffer to the C++ encoder + // without copying it to an intermediate byte array. + private val zeroCopyInput = StableRef.create(ZeroCopyInputSource(source)) + + // construct the pw_decoder_t by passing a pw_zero_copy_input_t that provides a bridge between + // the CodedInputStream and the given source buffer. it passes functions that call the respective + // ZeroCopyInputSource methods. + internal val raw = run { + if (source.exhausted()) { + error("Failed to create WireDecoder: provided buffer is empty") + } + + // construct the pw_zero_copy_input_t that functions as a bridge to the ZeroCopyInputSource + val zeroCopyCInput = cValue { + ctx = zeroCopyInput.asCPointer() + next = staticCFunction { ctx, data, size -> + ctx!!.asStableRef().get().next(data!!.reinterpret(), size!!.reinterpret()) + } + backUp = staticCFunction { ctx, count -> + ctx!!.asStableRef().get().backUp(count) + } + skip = staticCFunction { ctx, count -> + ctx!!.asStableRef().get().skip(count) + } + byteCount = staticCFunction { ctx -> + ctx!!.asStableRef().get().byteCount() + } + } + pw_decoder_new(zeroCopyCInput) + ?: error("Failed to create proto wire decoder") + } + + + override fun close() { + // delete the underlying decoder. + // this will also fix the position in the source buffer + // (done by deconstructor of CodedInputStream) + pw_decoder_delete(raw) + // close zero inputs on close + zeroCopyInput.get().close() + zeroCopyInput.dispose() + } + + override fun readTag(): KTag? { + val tag = pw_decoder_read_tag(raw) + return KTag.from(tag) + } + + override fun readBool(): Boolean? = memScoped { + val value = alloc() + if (pw_decoder_read_bool(raw, value.ptr)) { + return value.value + } + return null + } + + override fun readInt32(): Int? = memScoped { + val value = alloc() + if (pw_decoder_read_int32(raw, value.ptr)) { + return value.value + } + return null + } + + override fun readInt64(): Long? = memScoped { + val value = alloc() + if (pw_decoder_read_int64(raw, value.ptr)) { + return value.value + } + return null + } + + override fun readUInt32(): UInt? = memScoped { + val value = alloc() + if (pw_decoder_read_uint32(raw, value.ptr)) { + return value.value + } + return null + } + + override fun readUInt64(): ULong? = memScoped { + val value = alloc() + if (pw_decoder_read_uint64(raw, value.ptr)) { + return value.value + } + return null + } + + override fun readSInt32(): Int? = memScoped { + val value = alloc() + if (pw_decoder_read_sint32(raw, value.ptr)) { + return value.value + } + return null + } + + override fun readSInt64(): Long? = memScoped { + val value = alloc() + if (pw_decoder_read_sint64(raw, value.ptr)) { + return value.value + } + return null + } + + override fun readFixed32(): UInt? = memScoped { + val value = alloc() + if (pw_decoder_read_fixed32(raw, value.ptr)) { + return value.value + } + return null + } + + override fun readFixed64(): ULong? = memScoped { + val value = alloc() + if (pw_decoder_read_fixed64(raw, value.ptr)) { + return value.value + } + return null + } + + override fun readSFixed32(): Int? = memScoped { + val value = alloc() + if (pw_decoder_read_sfixed32(raw, value.ptr)) { + return value.value + } + return null + } + + override fun readSFixed64(): Long? = memScoped { + val value = alloc() + if (pw_decoder_read_sfixed64(raw, value.ptr)) { + return value.value + } + return null + } + + override fun readEnum(): Int? = memScoped { + val value = alloc() + if (pw_decoder_read_enum(raw, value.ptr)) { + return value.value + } + return null + } + + override fun readString(): String? = memScoped { + val str = alloc>() + val ok = pw_decoder_read_string(raw, str.ptr) + try { + if (!ok) return null + return pw_string_c_str(str.value)?.toKString() + } finally { + pw_string_delete(str.value) + } + } +} + +internal actual fun WireDecoder(source: Buffer): WireDecoder = WireDecoderNative(source) \ No newline at end of file diff --git a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.native.kt b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.native.kt new file mode 100644 index 000000000..1ac8086cb --- /dev/null +++ b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.native.kt @@ -0,0 +1,108 @@ +/* + * Copyright 2023-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.rpc.grpc.internal + +import kotlinx.cinterop.* +import kotlinx.io.Sink +import libprotowire.* +import kotlin.experimental.ExperimentalNativeApi +import kotlin.native.ref.createCleaner + + +@OptIn(ExperimentalForeignApi::class, ExperimentalNativeApi::class) +internal class WireEncoderNative(private val sink: Sink): WireEncoder { + /** + * The context object provides a stable reference to the kotlin context. + * This is required, as functions must be static and cannot capture environment references. + * With this context, the write callback (called by the pw_encoder_t) is able + * to write the data to the [sink]. + */ + private inner class Ctx { + fun write(buf: CPointer, size: Int): Boolean { + sink.writeFully(buf, 0L, size.toLong()) + return true + } + } + + // create context as a stable reference that can be passed to static function callback + private val context = StableRef.create(this.Ctx()) + // construct encoder with a callback that calls write() on this.context + internal val raw = run { + pw_encoder_new(context.asCPointer(), staticCFunction { ctx, buf, size -> + if (buf == null || ctx == null) { + return@staticCFunction false + } + ctx.asStableRef().get().write(buf.reinterpret(), size) + }) ?: error("Failed to create proto wire encoder") + } + + private val contextCleaner = createCleaner(context) { + it.dispose() + } + private val rawCleaner = createCleaner(raw) { + pw_encoder_delete(it) + } + + override fun writeBool(field: Int, value: Boolean): Boolean { + return pw_encoder_write_bool(raw, field, value) + } + + override fun writeInt32(fieldNr: Int, value: Int): Boolean { + return pw_encoder_write_int32(raw, fieldNr, value) + } + + override fun writeInt64(fieldNr: Int, value: Long): Boolean { + return pw_encoder_write_int64(raw, fieldNr, value) + } + + override fun writeUInt32(fieldNr: Int, value: UInt): Boolean { + return pw_encoder_write_uint32(raw, fieldNr, value) + } + + override fun writeUInt64(fieldNr: Int, value: ULong): Boolean { + return pw_encoder_write_uint64(raw, fieldNr, value) + } + + override fun writeSInt32(fieldNr: Int, value: Int): Boolean { + return pw_encoder_write_sint32(raw, fieldNr, value) + } + + override fun writeSInt64(fieldNr: Int, value: Long): Boolean { + return pw_encoder_write_sint64(raw, fieldNr, value) + } + + override fun writeFixed32(fieldNr: Int, value: UInt): Boolean { + return pw_encoder_write_fixed32(raw, fieldNr, value) + } + + override fun writeFixed64(fieldNr: Int, value: ULong): Boolean { + return pw_encoder_write_fixed64(raw, fieldNr, value) + } + + override fun writeSFixed32(fieldNr: Int, value: Int): Boolean { + return pw_encoder_write_sfixed32(raw, fieldNr, value) + } + + override fun writeSFixed64(fieldNr: Int, value: Long): Boolean { + return pw_encoder_write_sfixed64(raw, fieldNr, value) + } + + override fun writeEnum(fieldNr: Int, value: Int): Boolean { + return pw_encoder_write_enum(raw, fieldNr, value) + } + + override fun writeString(fieldNr: Int, value: String): Boolean { + val str = pw_string_new(value) ?: error("Failed to create string") + val result = pw_encoder_write_string(raw, fieldNr, str) + pw_string_delete(str) + return result; + } + + override fun flush() { + pw_encoder_flush(raw) + } +} + +internal actual fun WireEncoder(sink: Sink): WireEncoder = WireEncoderNative(sink) \ No newline at end of file diff --git a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/ZeroCopyInputSource.kt b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/ZeroCopyInputSource.kt new file mode 100644 index 000000000..f0b0b7a7f --- /dev/null +++ b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/ZeroCopyInputSource.kt @@ -0,0 +1,176 @@ +/* + * Copyright 2023-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.rpc.grpc.internal + +import kotlinx.cinterop.* +import kotlinx.io.Buffer +import kotlinx.io.EOFException +import kotlinx.io.InternalIoApi +import kotlinx.io.UnsafeIoApi +import kotlinx.io.unsafe.UnsafeBufferOperations + + +/** + * Handles (almost) zero-copy input operations on a [Buffer], allowing efficient transfer of data + * without creating intermediate copies. This class provides mechanisms to iterate over + * buffer data in a zero-copy manner and supports backing up, skipping, and advancing data. + * + * This class is intended for internal use only and specifically designed to be used + * as a bridge between [Buffer] and the C++ protobuf `ZeroCopyInputStream`. + * This implementation assumes that the data of a buffer segment is backed by a [ByteArray]. + * + * The inner [Buffer] MUST NOT be accessed while using the [ZeroCopyInputSource] as it highly + * depends on tracked state of [Buffer] internals. Each read or write to the underlying buffer + * will result in undefined behavior. + * Additionally, [ZeroCopyInputSource] is not thread-safe, so concurrent access might also + * result in undefined behavior. + * + * Unlike [Buffer.readByte], the [ZeroCopyInputSource.next] does directly consume the data + * in the [Buffer]. This has two reasons: + * 1. The underlying [ByteArray] must stay valid after the `next()` call + * 2. The [ZeroCopyInputSource.backUp] method might preserve bytes that where already read + * during the last [ZeroCopyInputSource.next] call. This method is required by the + * `ZeroCopyInputStream` interface of protobuf. + * Because of this, the inner [Buffer] is in an invalid read position during the use of + * [ZeroCopyInputSource]. After closing the [ZeroCopyInputSource] the inner [Buffer] + * is valid again. However, the buffer might be further advanced than expected, + * depending on whether the user called [backUp] for the unused bytes. + * + * The that memory received by a call to [ZeroCopyInputSource.next] is only valid until the next + * invocation of any method of [ZeroCopyInputSource]. + * + * @param inner The underlying [Buffer] to read data from. If must not be accessed while using + * [ZeroCopyInputSource]. + */ +@OptIn(ExperimentalForeignApi::class, InternalIoApi::class, UnsafeIoApi::class) +internal class ZeroCopyInputSource(private val inner: Buffer) : AutoCloseable { + + // number of bytes read since construction + private var byteCount = 0L + // the array segment that was read by the latest call to next() + // while it was already read by the ZeroCopyInputSource user, it is not yet + // released by the buffer. this is done by releaseLatestReadSegement + // which releases the segment in the inner buffer. + private var latestReadSegementArray: Pinned? = null + + /** + * Get access to a segment of continuous bytes in the underlying [Buffer]. + * The returned memory gets invalid with a call to `next(), backUp(), skip()` or `close()`. + * If the method returns `false`, if the inner buffer is exhausted. The `outData` and + * `outSize` remain unset in this case. + * + * @return false if the buffer is exhausted, otherwise true + */ + fun next(outData: CPointer>, outSize: CPointer): Boolean { + if (latestReadSegementArray != null) { + // if there is some unreleased segment array, we must release it first. + // this will advance the head of the buffer to the correct position. + releaseLatestReadSegment() + } + if (inner.exhausted()) { + return false + } + // perform access to the underlying array of the buffer's current segment + UnsafeBufferOperations.readFromHead(inner.buffer) { arr, start, end -> + check(latestReadSegementArray == null) { "currArr must be null at this point"} + // fix the array so it does not move in memory, which is important as we pass its + // memory address as a result to the caller. + latestReadSegementArray = arr.pin() + + val segmentSize = end - start + outData.pointed.value = latestReadSegementArray!!.addressOf(start) + outSize.pointed.value = segmentSize; + + byteCount += segmentSize; + + // we are not yet advancing the inner buffer head. + // this ensures that the segment array is not released by the buffer and remains valid + 0 + } + return true; + } + + /** + * Allows to replay [count] many bytes of the previously read segment. + * This is useful when writing procedures that are only supposed to read up + * to a certain point in the input, then return. If [next] returns a + * buffer that goes beyond what you wanted to read, you can use [backUp] + * to return to the point where you intended to finish. + * ```kt + * next(...) // access the current buffer segment + * backUp(10) // back up the last 10 bytes of the previous accessed segment + * next(...) // read the 10 last bytes of the previous accessed segment again + * ``` + * This is only possible if [next] was the last method called. + * + */ + fun backUp(count: Int) { + check(latestReadSegementArray != null) { "next() must be immediately before backUp()" } + releaseLatestReadSegment(count) + byteCount -= count; + } + + /** + * Skip [count] bytes of the buffer. + * @return `false` iff the buffer is exhausted before skipping completed, `true` otherwise + */ + fun skip(count: Int): Boolean { + if (latestReadSegementArray != null) { + releaseLatestReadSegment(count) + } + try { + byteCount += count + inner.skip(count.toLong()) + return true + } catch (_: EOFException) { + return false + } + } + + /** + * The number of bytes read since the object got created. + * If [backUp] is called, it will decrement the number of read bytes by the given amount. + */ + fun byteCount(): Long { + return byteCount + } + + /** + * Releases the latest read segment that was not yet released. + * It won't close the underlying [Buffer]. After closing this, the underlying [Buffer] is + * valid and can be used again. + * + * This [ZeroCopyInputSource] must not be used after closing it. + */ + override fun close() { + if (latestReadSegementArray != null) { + releaseLatestReadSegment() + } + } + + /** + * Releases the segment that was previously read using [next] but not yet released by the buffer. + * It also unpins it, so it can be collected by the GC. This must only be called if [next] was previously called. + * + * The [backUpCount] defines how many bytes of the segment should stay valid (not released). This is used by the + * [backUp] to allow users to replay reading of the latest read segment. + */ + private fun releaseLatestReadSegment(backUpCount: Int = 0) { + check(latestReadSegementArray != null) { "currArr must be not null" } + // the return value of the readFromHead defines the number of bytes that are getting released in the underlying + // buffer. + UnsafeBufferOperations.readFromHead(inner.buffer) { arr, start, end -> + check(latestReadSegementArray?.get() == arr) { + "array to advance must be the SAME as the currArr, was there some access to the underlying buffer?" } + // release the whole segmentSize - the backup count. + // prevent the value from being negative. + val read = maxOf(end - start - backUpCount, 0) + read + } + // remove tracking of the released segment + latestReadSegementArray?.unpin() + latestReadSegementArray = null; + } +} \ No newline at end of file diff --git a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/bufferUnsafeExtensions.kt b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/bufferUnsafeExtensions.kt new file mode 100644 index 000000000..fdfcff4d3 --- /dev/null +++ b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/bufferUnsafeExtensions.kt @@ -0,0 +1,30 @@ +/* + * Copyright 2023-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.rpc.grpc.internal + +import kotlinx.cinterop.* +import kotlinx.io.InternalIoApi +import kotlinx.io.Sink +import kotlinx.io.UnsafeIoApi +import kotlinx.io.unsafe.UnsafeBufferOperations +import platform.posix.memcpy + + +@OptIn(ExperimentalForeignApi::class, InternalIoApi::class, UnsafeIoApi::class) +public fun Sink.writeFully(buffer: CPointer, offset: Long, length: Long) { + var consumed = 0L + while (consumed < length) { + UnsafeBufferOperations.writeToTail(this.buffer, 1) { array, start, endExclusive -> + val size = minOf(length - consumed, (endExclusive - start).toLong()) + + array.usePinned { + memcpy(it.addressOf(start), buffer + offset + consumed, size.convert()) + } + + consumed += size + size.toInt() + } + } +} \ No newline at end of file diff --git a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/protowire/WireDecoder.kt b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/protowire/WireDecoder.kt deleted file mode 100644 index 905e7664e..000000000 --- a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/protowire/WireDecoder.kt +++ /dev/null @@ -1,142 +0,0 @@ -/* - * Copyright 2023-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. - */ - -package kotlinx.rpc.grpc.internal.protowire - -import kotlinx.cinterop.* -import libprotowire.* -import kotlin.experimental.ExperimentalNativeApi -import kotlin.native.ref.createCleaner - -@ExperimentalForeignApi -@OptIn(ExperimentalNativeApi::class) -internal class WireDecoder(buffer: UByteArray) { - internal val pinnedBuffer = buffer.pin() - internal val raw = pw_decoder_new(pinnedBuffer.addressOf(0), buffer.size.toUInt()) - ?: error("Failed to create proto wire decoder") - - init { - // free the encoder once garbage collector collects its pointer - createCleaner(raw) { - pw_decoder_delete(it) - } - createCleaner(pinnedBuffer) { - // it is not sure if this is needed (https://kotlinlang.slack.com/archives/C3SGXARS6/p1752851274402679) - // or if this is already done by the GC. - it.unpin() - } - } - - fun readTag(): KTag? { - val tag = pw_decoder_read_tag(raw) - return KTag.from(tag) - } - - fun readBool(): Boolean? = memScoped { - val value = alloc() - if (pw_decoder_read_bool(raw, value.ptr)) { - return value.value - } - return null - } - - fun readInt32(): Int? = memScoped { - val value = alloc() - if (pw_decoder_read_int32(raw, value.ptr)) { - return value.value - } - return null - } - - fun readInt64(): Long? = memScoped { - val value = alloc() - if (pw_decoder_read_int64(raw, value.ptr)) { - return value.value - } - return null - } - - fun readUInt32(): UInt? = memScoped { - val value = alloc() - if (pw_decoder_read_uint32(raw, value.ptr)) { - return value.value - } - return null - } - - fun readUInt64(): ULong? = memScoped { - val value = alloc() - if (pw_decoder_read_uint64(raw, value.ptr)) { - return value.value - } - return null - } - - fun readSInt32(): Int? = memScoped { - val value = alloc() - if (pw_decoder_read_sint32(raw, value.ptr)) { - return value.value - } - return null - } - - fun readSInt64(): Long? = memScoped { - val value = alloc() - if (pw_decoder_read_sint64(raw, value.ptr)) { - return value.value - } - return null - } - - fun readFixed32(): UInt? = memScoped { - val value = alloc() - if (pw_decoder_read_fixed32(raw, value.ptr)) { - return value.value - } - return null - } - - fun readFixed64(): ULong? = memScoped { - val value = alloc() - if (pw_decoder_read_fixed64(raw, value.ptr)) { - return value.value - } - return null - } - - fun readSFixed32(): Int? = memScoped { - val value = alloc() - if (pw_decoder_read_sfixed32(raw, value.ptr)) { - return value.value - } - return null - } - - fun readSFixed64(): Long? = memScoped { - val value = alloc() - if (pw_decoder_read_sfixed64(raw, value.ptr)) { - return value.value - } - return null - } - - fun readEnum(): Int? = memScoped { - val value = alloc() - if (pw_decoder_read_enum(raw, value.ptr)) { - return value.value - } - return null - } - - fun readString(): String? = memScoped { - val str = alloc>() - val ok = pw_decoder_read_string(raw, str.ptr) - try { - if (!ok) return null - return pw_string_c_str(str.value)?.toKString() - } finally { - pw_string_delete(str.value) - } - } -} diff --git a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/protowire/WireEncoder.kt b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/protowire/WireEncoder.kt deleted file mode 100644 index 53c753949..000000000 --- a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/protowire/WireEncoder.kt +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright 2023-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. - */ - -package kotlinx.rpc.grpc.internal.protowire - -import kotlinx.cinterop.ExperimentalForeignApi -import kotlinx.cinterop.Pinned -import kotlinx.cinterop.addressOf -import kotlinx.cinterop.pin -import libprotowire.* -import kotlin.experimental.ExperimentalNativeApi -import kotlin.native.ref.createCleaner - -@ExperimentalForeignApi -@OptIn(ExperimentalNativeApi::class) -internal class WireEncoder { - internal val buffer: Pinned = UByteArray(1024).pin() - internal val raw = pw_encoder_new(buffer.addressOf(0), 1024u) - ?: error("Failed to create proto wire encoder") - - init { - // free the encoder once garbage collector collects its pointer - createCleaner(raw) { - pw_encoder_delete(it) - } - } - - fun writeBool(field: Int, value: Boolean): Boolean { - return pw_encoder_write_bool(raw, field, value) - } - - fun writeInt32(fieldNr: Int, value: Int): Boolean { - return pw_encoder_write_int32(raw, fieldNr, value) - } - - fun writeInt64(fieldNr: Int, value: Long): Boolean { - return pw_encoder_write_int64(raw, fieldNr, value) - } - - fun writeUInt32(fieldNr: Int, value: UInt): Boolean { - return pw_encoder_write_uint32(raw, fieldNr, value) - } - - fun writeUInt64(fieldNr: Int, value: ULong): Boolean { - return pw_encoder_write_uint64(raw, fieldNr, value) - } - - fun writeSInt32(fieldNr: Int, value: Int): Boolean { - return pw_encoder_write_sint32(raw, fieldNr, value) - } - - fun writeSInt64(fieldNr: Int, value: Long): Boolean { - return pw_encoder_write_sint64(raw, fieldNr, value) - } - - fun writeFixed32(fieldNr: Int, value: UInt): Boolean { - return pw_encoder_write_fixed32(raw, fieldNr, value) - } - - fun writeFixed64(fieldNr: Int, value: ULong): Boolean { - return pw_encoder_write_fixed64(raw, fieldNr, value) - } - - fun writeSFixed32(fieldNr: Int, value: Int): Boolean { - return pw_encoder_write_sfixed32(raw, fieldNr, value) - } - - fun writeSFixed64(fieldNr: Int, value: Long): Boolean { - return pw_encoder_write_sfixed64(raw, fieldNr, value) - } - - fun writeEnum(fieldNr: Int, value: Int): Boolean { - return pw_encoder_write_enum(raw, fieldNr, value) - } - - fun writeString(field: Int, value: String): Boolean { - val str = pw_string_new(value) ?: error("Failed to create string") - val result = pw_encoder_write_string(raw, field, str) - pw_string_delete(str) - return result; - } - -} \ No newline at end of file diff --git a/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/ProtoWireTest.kt b/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/ProtoWireTest.kt index 41df1227f..f7ae4510f 100644 --- a/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/ProtoWireTest.kt +++ b/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/ProtoWireTest.kt @@ -5,9 +5,7 @@ package kotlinx.rpc.grpc.internal import kotlinx.cinterop.ExperimentalForeignApi -import kotlinx.rpc.grpc.internal.protowire.WireDecoder -import kotlinx.rpc.grpc.internal.protowire.WireEncoder -import kotlinx.rpc.grpc.internal.protowire.WireType +import kotlinx.io.Buffer import kotlin.experimental.ExperimentalNativeApi import kotlin.test.Test @@ -19,12 +17,13 @@ class ProtoWireTest { fun testEncodeDecodeBool() { val fieldNr = 3; - val encoder = WireEncoder() + val buffer = Buffer() + val encoder = WireEncoderNative(buffer) encoder.writeBool(fieldNr, true) encoder.writeString(4, "Hello Test") + encoder.flush() - val encodedBuffer = encoder.buffer.get() - val decoder = WireDecoder(encodedBuffer) + val decoder = WireDecoderNative(buffer) val t1 = decoder.readTag()!! check(t1.wireType == WireType.VARINT) @@ -39,5 +38,8 @@ class ProtoWireTest { val str = decoder.readString()!! check(str == "Hello Test") + + decoder.close() + assert(buffer.exhausted()) } } diff --git a/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/ZeroCopyInputSourceTest.kt b/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/ZeroCopyInputSourceTest.kt new file mode 100644 index 000000000..1c5af05ee --- /dev/null +++ b/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/ZeroCopyInputSourceTest.kt @@ -0,0 +1,82 @@ +/* + * Copyright 2023-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.rpc.grpc.internal + +import kotlinx.cinterop.ByteVar +import kotlinx.cinterop.CPointerVar +import kotlinx.cinterop.ExperimentalForeignApi +import kotlinx.cinterop.IntVar +import kotlinx.cinterop.addressOf +import kotlinx.cinterop.alloc +import kotlinx.cinterop.memScoped +import kotlinx.cinterop.ptr +import kotlinx.cinterop.usePinned +import kotlinx.cinterop.value +import kotlinx.io.Buffer +import platform.posix.memcpy +import kotlin.experimental.ExperimentalNativeApi +import kotlin.test.Test +import kotlin.test.assertEquals + +@OptIn(ExperimentalForeignApi::class, ExperimentalNativeApi::class) +class ZeroCopyInputSourceTest { + + + @Test + fun simpleTest() { + + + val buffer = Buffer() + fillWithChunks(buffer, 1000, 10) + + val zeroCopyInputSource = ZeroCopyInputSource(buffer) + + var i = 0 + var count = 0 + while (true) { + val read = zeroCopyInputSource.nextIntoArray() + println("$i reads: ${read.size}") + if (read.isEmpty()) { + break + } + if (read.size >= 10) { + val toBackup = (read.size - 1) % ((i + 1) * 100) + count -= toBackup + zeroCopyInputSource.backUp( toBackup) + } + count += read.size + i++ + } + + + assertEquals(10000, zeroCopyInputSource.byteCount()) + assertEquals(10000, count) + } + + private fun fillWithChunks(buffer: Buffer, numberOfChunks: Int, chunkSize: Int) { + repeat(numberOfChunks) { i -> + buffer.write(ByteArray(chunkSize) { i.toByte() }) + } + + } + +} + + +@OptIn(ExperimentalForeignApi::class) +private fun ZeroCopyInputSource.nextIntoArray(): ByteArray = memScoped { + val data = alloc>() + val size = alloc() + + if (!next(data.ptr, size.ptr)) { + return ByteArray(0) + } + + val result = ByteArray(size.value) + result.usePinned { + memcpy(it.addressOf(0), data.value, size.value.toULong()) + } + result +} \ No newline at end of file diff --git a/grpc/grpc-core/src/wasmJsMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.wasmJs.kt b/grpc/grpc-core/src/wasmJsMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.wasmJs.kt new file mode 100644 index 000000000..f56885157 --- /dev/null +++ b/grpc/grpc-core/src/wasmJsMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.wasmJs.kt @@ -0,0 +1,12 @@ +/* + * Copyright 2023-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.rpc.grpc.internal + +import kotlinx.io.Buffer +import kotlinx.io.Source + +internal actual fun WireDecoder(source: Buffer): WireDecoder { + TODO("Not yet implemented") +} \ No newline at end of file diff --git a/grpc/grpc-core/src/wasmJsMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.wasmJs.kt b/grpc/grpc-core/src/wasmJsMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.wasmJs.kt new file mode 100644 index 000000000..00c0b3246 --- /dev/null +++ b/grpc/grpc-core/src/wasmJsMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.wasmJs.kt @@ -0,0 +1,11 @@ +/* + * Copyright 2023-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.rpc.grpc.internal + +import kotlinx.io.Sink + +internal actual fun WireEncoder(sink: Sink): WireEncoder { + TODO("Not yet implemented") +} \ No newline at end of file diff --git a/grpc/grpcpp-c/BUILD.bazel b/grpc/grpcpp-c/BUILD.bazel index da41f770c..566d973df 100644 --- a/grpc/grpcpp-c/BUILD.bazel +++ b/grpc/grpcpp-c/BUILD.bazel @@ -1,6 +1,14 @@ load("@emsdk//emscripten_toolchain:wasm_rules.bzl", "wasm_cc_binary") load("@rules_cc//cc:defs.bzl", "cc_library") +cc_binary( + name = "testdemo", + srcs = ["src/main.cpp"], + deps = [ + ":protowire", + ], +) + cc_static_library( name = "grpcpp_c_static", deps = [ @@ -16,9 +24,12 @@ cc_library( includes = ["include"], visibility = ["//visibility:public"], deps = [ + # TODO: Reduce the dependencies and only use required once "@com_github_grpc_grpc//:channelz", "@com_github_grpc_grpc//:generic_stub", + "@com_github_grpc_grpc//:grpc++", "@com_github_grpc_grpc//:grpc_credentials_util", + "@com_google_protobuf//:protobuf", ], ) diff --git a/grpc/grpcpp-c/include/protowire.h b/grpc/grpcpp-c/include/protowire.h index 6692269e8..5657bb0c8 100644 --- a/grpc/grpcpp-c/include/protowire.h +++ b/grpc/grpcpp-c/include/protowire.h @@ -19,8 +19,9 @@ extern "C" { typedef struct pw_encoder pw_encoder_t; - pw_encoder_t * pw_encoder_new(uint8_t *buf, uint32_t cap); + pw_encoder_t *pw_encoder_new(void* ctx, bool (*)(void* ctx, const void* buf, int size)); void pw_encoder_delete(pw_encoder_t *self); + bool pw_encoder_flush(pw_encoder_t *self); bool pw_encoder_write_bool(pw_encoder_t *self, int field_no, bool value); bool pw_encoder_write_int32(pw_encoder_t *self, int field_no, int32_t value); @@ -45,7 +46,15 @@ extern "C" { typedef struct pw_decoder pw_decoder_t; - pw_decoder_t * pw_decoder_new(uint8_t *buf, uint32_t cap); + typedef struct pw_zero_copy_input { + void *ctx; + bool (*next)(void *ctx, const void **data, int *size); + void (*backUp)(void *ctx, int size); + bool (*skip)(void *ctx, int size); + int64_t (*byteCount)(void *ctx); + } pw_zero_copy_input_t; + + pw_decoder_t * pw_decoder_new(pw_zero_copy_input_t zero_copy_input); void pw_decoder_delete(pw_decoder_t *self); uint32_t pw_decoder_read_tag(pw_decoder_t *self); diff --git a/grpc/grpcpp-c/src/protowire.cpp b/grpc/grpcpp-c/src/protowire.cpp index 652352fac..0f2eeab08 100644 --- a/grpc/grpcpp-c/src/protowire.cpp +++ b/grpc/grpcpp-c/src/protowire.cpp @@ -4,37 +4,86 @@ #include "protowire.h" +#include + #include "src/google/protobuf/io/zero_copy_stream_impl_lite.h" #include "src/google/protobuf/io/coded_stream.h" #include "src/google/protobuf/wire_format_lite.h" namespace pb = google::protobuf; - typedef pb::internal::WireFormatLite WireFormatLite; +namespace protowire { +class SinkStream final : public pb::io::CopyingOutputStream { +public: + SinkStream(void *ctx, bool(*sink)(void *ctx, const void *buffer, int size)) + : ctx(ctx), + sink(sink) { + } + + bool Write(const void *buffer, int size) override { + return sink(ctx, buffer, size); + } + +private: + void *ctx; + bool (*sink)(void *ctx, const void *buffer, int size); +}; + +class SourceStream final : public pb::io::ZeroCopyInputStream { +public: + explicit SourceStream(const pw_zero_copy_input_t &input) + : input(input) { + } + + bool Next(const void **data, int *size) override { + return input.next(input.ctx, data, size); + }; + + void BackUp(int count) override { + return input.backUp(input.ctx, count); + }; + + bool Skip(int count) override { + return input.skip(input.ctx, count); + }; + + int64_t ByteCount() const override { + return input.byteCount(input.ctx); + }; + +private: + pw_zero_copy_input_t input; +}; + +} + struct pw_string { std::string str; }; struct pw_encoder { - pb::io::ArrayOutputStream aos; + protowire::SinkStream sinkStream; + pb::io::CopyingOutputStreamAdaptor cosa; pb::io::CodedOutputStream cos; - pw_encoder(uint8_t* buf, int size) - : aos(buf, size), - cos(&aos) {} + explicit pw_encoder(protowire::SinkStream sink) + : sinkStream(std::move(sink)), + cosa(&sinkStream), + cos(&cosa) {} }; struct pw_decoder { - pb::io::ArrayInputStream ais; + protowire::SourceStream ss; pb::io::CodedInputStream cis; - pw_decoder(uint8_t* buf, int size) - : ais(buf, size), - cis(&ais) {} + pw_decoder(pw_zero_copy_input_t input) + : ss(input), + cis(&ss) {} }; + extern "C" { pw_string_t *pw_string_new(const char *str) { @@ -47,13 +96,21 @@ extern "C" { return self->str.c_str(); } - pw_encoder_t * pw_encoder_new(uint8_t *buf, uint32_t cap) { - return new pw_encoder_t(buf, cap); + pw_encoder_t *pw_encoder_new(void* ctx, bool (* sink_fn)(void* ctx, const void* buf, int size)) { + auto sink = protowire::SinkStream(ctx, sink_fn); + return new pw_encoder(std::move(sink)); } void pw_encoder_delete(pw_encoder_t *self) { delete self; } + bool pw_encoder_flush(pw_encoder_t *self) { + self->cos.Trim(); + if (!self->cosa.Flush()) { + return false; + } + return !self->cos.HadError(); + } // check if there was an error static bool check(pw_encoder_t *self) { @@ -89,8 +146,8 @@ extern "C" { } - pw_decoder_t *pw_decoder_new(uint8_t *buf, uint32_t cap) { - return new pw_decoder_t(buf, cap); + pw_decoder_t *pw_decoder_new(pw_zero_copy_input_t zero_copy_input) { + return new pw_decoder_t(zero_copy_input); } void pw_decoder_delete(pw_decoder_t *self) { diff --git a/versions-root/libs.versions.toml b/versions-root/libs.versions.toml index 8c7137f4a..49b12aac5 100644 --- a/versions-root/libs.versions.toml +++ b/versions-root/libs.versions.toml @@ -20,6 +20,7 @@ junit5 = "5.13.2" intellij = "241.19416.19" gradle-doctor = "0.11.0" kotlinx-browser = "0.3" +kotlinx-io = "0.8.0" dokka = "2.0.0" puppeteer = "24.9.0" atomicfu = "0.29.0" @@ -59,6 +60,7 @@ kotlin-compiler-test-framework = { module = "org.jetbrains.kotlin:kotlin-compile serialization-plugin = { module = "org.jetbrains.kotlin:kotlin-serialization-compiler-plugin", version.ref = "kotlin-compiler" } serialization-plugin-forIde = { module = "org.jetbrains.kotlin:kotlinx-serialization-compiler-plugin-for-ide", version.ref = "kotlin-compiler" } kotlinx-browser = { module = "org.jetbrains.kotlinx:kotlinx-browser", version.ref = "kotlinx-browser" } +kotlinx-io-core = { module = "org.jetbrains.kotlinx:kotlinx-io-core", version.ref = "kotlinx-io" } # serialization serialization-core = { module = "org.jetbrains.kotlinx:kotlinx-serialization-core", version.ref = "serialization" } From bd75b580b5d80f39b86f7f53daf1409f5a1cb2c5 Mon Sep 17 00:00:00 2001 From: Johannes Zottele Date: Wed, 23 Jul 2025 15:41:45 +0200 Subject: [PATCH 05/16] grpc-native: Add ZeroCopyInputSourceTest.kt and WireCodecTest.kt Signed-off-by: Johannes Zottele --- .../kotlin/kotlinx/rpc/grpc/internal/KTag.kt | 24 +- .../kotlinx/rpc/grpc/internal/WireDecoder.kt | 16 + .../rpc/grpc/internal/WireDecoder.native.kt | 4 - .../rpc/grpc/internal/ZeroCopyInputSource.kt | 25 +- .../rpc/grpc/internal/ProtoWireTest.kt | 45 -- .../rpc/grpc/internal/WireCodecTest.kt | 556 ++++++++++++++++++ .../grpc/internal/ZeroCopyInputSourceTest.kt | 286 ++++++++- grpc/grpcpp-c/include/protowire.h | 3 + grpc/grpcpp-c/src/protowire.cpp | 9 +- 9 files changed, 885 insertions(+), 83 deletions(-) delete mode 100644 grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/ProtoWireTest.kt create mode 100644 grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/WireCodecTest.kt diff --git a/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/KTag.kt b/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/KTag.kt index c77fdfd50..e4c7e6288 100644 --- a/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/KTag.kt +++ b/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/KTag.kt @@ -14,19 +14,31 @@ internal enum class WireType { } internal data class KTag(val fieldNr: Int, val wireType: WireType) { + + init { + check(isValidFieldNr(fieldNr)) { "Invalid field number: $fieldNr" } + } + companion object { // Number of bits in a tag which identify the wire type. - const val K_TAG_TYPE_BITS: UInt = 3u; + const val K_TAG_TYPE_BITS: Int = 3; // Mask for those bits. (just 0b111) - val K_TAG_TYPE_MASK: UInt = (1u shl K_TAG_TYPE_BITS.toInt()) - 1u + val K_TAG_TYPE_MASK: UInt = (1u shl K_TAG_TYPE_BITS) - 1u } } internal fun KTag.Companion.from(rawKTag: UInt): KTag? { - val type = rawKTag and K_TAG_TYPE_MASK - val field = rawKTag shr K_TAG_TYPE_BITS.toInt() - if (type >= WireType.entries.size.toUInt()) { + val type = (rawKTag and K_TAG_TYPE_MASK).toInt() + val field = (rawKTag shr K_TAG_TYPE_BITS).toInt() + if (!isValidFieldNr(field)) { return null } - return KTag(field.toInt(), WireType.entries[type.toInt()]) + if (type >= WireType.entries.size) { + return null + } + return KTag(field, WireType.entries[type]) +} + +internal fun KTag.Companion.isValidFieldNr(fieldNr: Int): Boolean { + return 1 <= fieldNr && fieldNr <= 536_870_911 } \ No newline at end of file diff --git a/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.kt b/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.kt index e3d7f6cbb..3719c4058 100644 --- a/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.kt +++ b/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.kt @@ -11,6 +11,22 @@ import kotlinx.io.Buffer * * If one `read*()` method returns `null`, decoding the data failed and no further * decoding can be done. + * + * NOTE: If the value of a `read*()` method is non-null, it doesn't mean that the + * value is correctly decoded. E.g., the following test will pass: + * ```kt + * val fieldNr = 1 + * val buffer = Buffer() + * + * val encoder = WireEncoder(buffer) + * assertTrue(encoder.writeInt32(fieldNr, 12312)) + * encoder.flush() + * + * WireDecoder(buffer).use { decoder -> + * decoder.readTag() + * assertNotNull(decoder.readBool()) + * } + * ``` */ internal interface WireDecoder: AutoCloseable { fun readTag(): KTag? diff --git a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.native.kt b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.native.kt index 3e6b2428f..3aba74cb4 100644 --- a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.native.kt +++ b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.native.kt @@ -20,10 +20,6 @@ internal class WireDecoderNative(private val source: Buffer): WireDecoder { // the CodedInputStream and the given source buffer. it passes functions that call the respective // ZeroCopyInputSource methods. internal val raw = run { - if (source.exhausted()) { - error("Failed to create WireDecoder: provided buffer is empty") - } - // construct the pw_zero_copy_input_t that functions as a bridge to the ZeroCopyInputSource val zeroCopyCInput = cValue { ctx = zeroCopyInput.asCPointer() diff --git a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/ZeroCopyInputSource.kt b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/ZeroCopyInputSource.kt index f0b0b7a7f..e20df0607 100644 --- a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/ZeroCopyInputSource.kt +++ b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/ZeroCopyInputSource.kt @@ -54,6 +54,7 @@ internal class ZeroCopyInputSource(private val inner: Buffer) : AutoCloseable { // released by the buffer. this is done by releaseLatestReadSegement // which releases the segment in the inner buffer. private var latestReadSegementArray: Pinned? = null + private var closed = false; /** * Get access to a segment of continuous bytes in the underlying [Buffer]. @@ -64,6 +65,7 @@ internal class ZeroCopyInputSource(private val inner: Buffer) : AutoCloseable { * @return false if the buffer is exhausted, otherwise true */ fun next(outData: CPointer>, outSize: CPointer): Boolean { + check(!closed) { "ZeroCopyInputSource has already been closed." } if (latestReadSegementArray != null) { // if there is some unreleased segment array, we must release it first. // this will advance the head of the buffer to the correct position. @@ -105,10 +107,14 @@ internal class ZeroCopyInputSource(private val inner: Buffer) : AutoCloseable { * ``` * This is only possible if [next] was the last method called. * + * @throws IllegalStateException if [count] is greater than size of the last read segment (retrieved from [next]). + * */ fun backUp(count: Int) { + check(!closed) { "ZeroCopyInputSource has already been closed." } check(latestReadSegementArray != null) { "next() must be immediately before backUp()" } - releaseLatestReadSegment(count) + val readBytes = releaseLatestReadSegment(count) + check(readBytes >= 0) { "backUp() must not be called more than the number of bytes that were read in next()" } byteCount -= count; } @@ -117,8 +123,9 @@ internal class ZeroCopyInputSource(private val inner: Buffer) : AutoCloseable { * @return `false` iff the buffer is exhausted before skipping completed, `true` otherwise */ fun skip(count: Int): Boolean { + check(!closed) { "ZeroCopyInputSource has already been closed." } if (latestReadSegementArray != null) { - releaseLatestReadSegment(count) + releaseLatestReadSegment() } try { byteCount += count @@ -148,6 +155,7 @@ internal class ZeroCopyInputSource(private val inner: Buffer) : AutoCloseable { if (latestReadSegementArray != null) { releaseLatestReadSegment() } + closed = true; } /** @@ -156,21 +164,28 @@ internal class ZeroCopyInputSource(private val inner: Buffer) : AutoCloseable { * * The [backUpCount] defines how many bytes of the segment should stay valid (not released). This is used by the * [backUp] to allow users to replay reading of the latest read segment. + * If the [backUpCount] is greater than the segment size, 0 bytes are read. + * + * @return number of bytes released, based on [backUpCount]. This value might be negative + * if the [backUpCount] is greater than the latest read segment (indicates a user side error). */ - private fun releaseLatestReadSegment(backUpCount: Int = 0) { + private fun releaseLatestReadSegment(backUpCount: Int = 0): Int { check(latestReadSegementArray != null) { "currArr must be not null" } + var readBytes: Int // the return value of the readFromHead defines the number of bytes that are getting released in the underlying // buffer. UnsafeBufferOperations.readFromHead(inner.buffer) { arr, start, end -> check(latestReadSegementArray?.get() == arr) { "array to advance must be the SAME as the currArr, was there some access to the underlying buffer?" } // release the whole segmentSize - the backup count. + readBytes = end - start - backUpCount // prevent the value from being negative. - val read = maxOf(end - start - backUpCount, 0) - read + val safeReadBytes = maxOf(readBytes, 0) + safeReadBytes } // remove tracking of the released segment latestReadSegementArray?.unpin() latestReadSegementArray = null; + return readBytes } } \ No newline at end of file diff --git a/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/ProtoWireTest.kt b/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/ProtoWireTest.kt deleted file mode 100644 index f7ae4510f..000000000 --- a/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/ProtoWireTest.kt +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright 2023-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. - */ - -package kotlinx.rpc.grpc.internal - -import kotlinx.cinterop.ExperimentalForeignApi -import kotlinx.io.Buffer -import kotlin.experimental.ExperimentalNativeApi -import kotlin.test.Test - -@OptIn(ExperimentalForeignApi::class) -class ProtoWireTest { - - @OptIn(ExperimentalNativeApi::class) - @Test - fun testEncodeDecodeBool() { - val fieldNr = 3; - - val buffer = Buffer() - val encoder = WireEncoderNative(buffer) - encoder.writeBool(fieldNr, true) - encoder.writeString(4, "Hello Test") - encoder.flush() - - val decoder = WireDecoderNative(buffer) - - val t1 = decoder.readTag()!! - check(t1.wireType == WireType.VARINT) - check(t1.fieldNr == fieldNr) - - val bool = decoder.readBool()!! - check(bool) - - val t2 = decoder.readTag()!! - check(t2.wireType == WireType.LENGTH_DELIMITED) - check(t2.fieldNr == 4) - - val str = decoder.readString()!! - check(str == "Hello Test") - - decoder.close() - assert(buffer.exhausted()) - } -} diff --git a/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/WireCodecTest.kt b/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/WireCodecTest.kt new file mode 100644 index 000000000..c7651ae24 --- /dev/null +++ b/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/WireCodecTest.kt @@ -0,0 +1,556 @@ +/* + * Copyright 2023-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.rpc.grpc.internal + +import kotlinx.cinterop.ExperimentalForeignApi +import kotlinx.io.Buffer +import kotlin.experimental.ExperimentalNativeApi +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertNotNull +import kotlin.test.assertNull +import kotlin.test.assertTrue + +// TODO: Move this into the commonTest +@OptIn(ExperimentalForeignApi::class, ExperimentalNativeApi::class) +class WireCodecTest { + + @Test + fun testBoolEncodeDecode() { + val fieldNr = 3 + val buffer = Buffer() + + val encoder = WireEncoder(buffer) + assertTrue(encoder.writeBool(fieldNr, true)) + encoder.flush() + + val decoder = WireDecoder(buffer) + + val tag = decoder.readTag() + assertNotNull(tag) + assertEquals(WireType.VARINT, tag.wireType) + assertEquals(fieldNr, tag.fieldNr) + + val value = decoder.readBool() + assertNotNull(value) + assertTrue(value) + + decoder.close() + assertTrue(buffer.exhausted()) + } + + @Test + fun testInt32EncodeDecode() { + val fieldNr = 5 + val testValue = 42 + val buffer = Buffer() + + val encoder = WireEncoder(buffer) + assertTrue(encoder.writeInt32(fieldNr, testValue)) + encoder.flush() + + val decoder = WireDecoder(buffer) + + val tag = decoder.readTag() + assertNotNull(tag) + assertEquals(WireType.VARINT, tag.wireType) + assertEquals(fieldNr, tag.fieldNr) + + val value = decoder.readInt32() + assertNotNull(value) + assertEquals(testValue, value) + + decoder.close() + assertTrue(buffer.exhausted()) + } + + @Test + fun testInt64EncodeDecode() { + val fieldNr = 6 + val testValue = Long.MAX_VALUE + val buffer = Buffer() + + val encoder = WireEncoder(buffer) + assertTrue(encoder.writeInt64(fieldNr, testValue)) + encoder.flush() + + val decoder = WireDecoder(buffer) + + val tag = decoder.readTag() + assertNotNull(tag) + assertEquals(WireType.VARINT, tag.wireType) + assertEquals(fieldNr, tag.fieldNr) + + val value = decoder.readInt64() + assertNotNull(value) + assertEquals(testValue, value) + + decoder.close() + assertTrue(buffer.exhausted()) + } + + @Test + fun testUInt32EncodeDecode() { + val fieldNr = 7 + val testValue = UInt.MAX_VALUE + val buffer = Buffer() + + val encoder = WireEncoder(buffer) + assertTrue(encoder.writeUInt32(fieldNr, testValue)) + encoder.flush() + + val decoder = WireDecoder(buffer) + + val tag = decoder.readTag() + assertNotNull(tag) + assertEquals(WireType.VARINT, tag.wireType) + assertEquals(fieldNr, tag.fieldNr) + + val value = decoder.readUInt32() + assertNotNull(value) + assertEquals(testValue, value) + + decoder.close() + assertTrue(buffer.exhausted()) + } + + @Test + fun testUInt64EncodeDecode() { + val fieldNr = 8 + val testValue = ULong.MAX_VALUE + val buffer = Buffer() + + val encoder = WireEncoder(buffer) + assertTrue(encoder.writeUInt64(fieldNr, testValue)) + encoder.flush() + + val decoder = WireDecoder(buffer) + + val tag = decoder.readTag() + assertNotNull(tag) + assertEquals(WireType.VARINT, tag.wireType) + assertEquals(fieldNr, tag.fieldNr) + + val value = decoder.readUInt64() + assertNotNull(value) + assertEquals(testValue, value) + + decoder.close() + assertTrue(buffer.exhausted()) + } + + @Test + fun testSInt32EncodeDecode() { + val fieldNr = 9 + val testValue = Int.MIN_VALUE + val buffer = Buffer() + + val encoder = WireEncoder(buffer) + assertTrue(encoder.writeSInt32(fieldNr, testValue)) + encoder.flush() + + val decoder = WireDecoder(buffer) + + val tag = decoder.readTag() + assertNotNull(tag) + assertEquals(WireType.VARINT, tag.wireType) + assertEquals(fieldNr, tag.fieldNr) + + val value = decoder.readSInt32() + assertNotNull(value) + assertEquals(testValue, value) + + decoder.close() + assertTrue(buffer.exhausted()) + } + + @Test + fun testSInt64EncodeDecode() { + val fieldNr = 10 + val testValue = Long.MIN_VALUE // Min long value + val buffer = Buffer() + + val encoder = WireEncoder(buffer) + assertTrue(encoder.writeSInt64(fieldNr, testValue)) + encoder.flush() + + val decoder = WireDecoder(buffer) + + val tag = decoder.readTag() + assertNotNull(tag) + assertEquals(WireType.VARINT, tag.wireType) + assertEquals(fieldNr, tag.fieldNr) + + val value = decoder.readSInt64() + assertNotNull(value) + assertEquals(testValue, value) + + decoder.close() + assertTrue(buffer.exhausted()) + } + + @Test + fun testFixed32EncodeDecode() { + val fieldNr = 11 + val testValue = UInt.MAX_VALUE + val buffer = Buffer() + + val encoder = WireEncoder(buffer) + assertTrue(encoder.writeFixed32(fieldNr, testValue)) + encoder.flush() + + val decoder = WireDecoder(buffer) + + val tag = decoder.readTag() + assertNotNull(tag) + assertEquals(WireType.FIXED32, tag.wireType) + assertEquals(fieldNr, tag.fieldNr) + + val value = decoder.readFixed32() + assertNotNull(value) + assertEquals(testValue, value) + + decoder.close() + assertTrue(buffer.exhausted()) + } + + @Test + fun testFixed64EncodeDecode() { + val fieldNr = 12 + val testValue = ULong.MAX_VALUE + val buffer = Buffer() + + val encoder = WireEncoder(buffer) + assertTrue(encoder.writeFixed64(fieldNr, testValue)) + encoder.flush() + + val decoder = WireDecoder(buffer) + + val tag = decoder.readTag() + assertNotNull(tag) + assertEquals(WireType.FIXED64, tag.wireType) + assertEquals(fieldNr, tag.fieldNr) + + val value = decoder.readFixed64() + assertNotNull(value) + assertEquals(testValue, value) + + decoder.close() + assertTrue(buffer.exhausted()) + } + + @Test + fun testSFixed32EncodeDecode() { + val fieldNr = 13 + val testValue = Int.MIN_VALUE + val buffer = Buffer() + + val encoder = WireEncoder(buffer) + assertTrue(encoder.writeSFixed32(fieldNr, testValue)) + encoder.flush() + + val decoder = WireDecoder(buffer) + + val tag = decoder.readTag() + assertNotNull(tag) + assertEquals(WireType.FIXED32, tag.wireType) + assertEquals(fieldNr, tag.fieldNr) + + val value = decoder.readSFixed32() + assertNotNull(value) + assertEquals(testValue, value) + + decoder.close() + assertTrue(buffer.exhausted()) + } + + @Test + fun testSFixed64EncodeDecode() { + val fieldNr = 14 + val testValue = Long.MIN_VALUE + val buffer = Buffer() + + val encoder = WireEncoder(buffer) + assertTrue(encoder.writeSFixed64(fieldNr, testValue)) + encoder.flush() + + val decoder = WireDecoder(buffer) + + val tag = decoder.readTag() + assertNotNull(tag) + assertEquals(WireType.FIXED64, tag.wireType) + assertEquals(fieldNr, tag.fieldNr) + + val value = decoder.readSFixed64() + assertNotNull(value) + assertEquals(testValue, value) + + decoder.close() + assertTrue(buffer.exhausted()) + } + + @Test + fun testEnumEncodeDecode() { + val fieldNr = 15 + val testValue = 42 + val buffer = Buffer() + + val encoder = WireEncoder(buffer) + assertTrue(encoder.writeEnum(fieldNr, testValue)) + encoder.flush() + + val decoder = WireDecoder(buffer) + + val tag = decoder.readTag() + assertNotNull(tag) + assertEquals(WireType.VARINT, tag.wireType) + assertEquals(fieldNr, tag.fieldNr) + + val value = decoder.readEnum() + assertNotNull(value) + assertEquals(testValue, value) + + decoder.close() + assertTrue(buffer.exhausted()) + } + + @Test + fun testStringEncodeDecode() { + val fieldNr = 16 + val testValue = "Hello, World!" + val buffer = Buffer() + + val encoder = WireEncoder(buffer) + assertTrue(encoder.writeString(fieldNr, testValue)) + encoder.flush() + + val decoder = WireDecoder(buffer) + + val tag = decoder.readTag() + assertNotNull(tag) + assertEquals(WireType.LENGTH_DELIMITED, tag.wireType) + assertEquals(fieldNr, tag.fieldNr) + + val value = decoder.readString() + assertNotNull(value) + assertEquals(testValue, value) + + decoder.close() + assertTrue(buffer.exhausted()) + } + + @Test + fun testEmptyBufferDecoding() { + val buffer = Buffer() + + val decoder = WireDecoder(buffer) + assertNull(decoder.readTag()) + assertNull(decoder.readBool()) + assertNull(decoder.readInt32()) + assertNull(decoder.readInt64()) + assertNull(decoder.readSInt32()) + assertNull(decoder.readSInt64()) + assertNull(decoder.readUInt32()) + assertNull(decoder.readUInt64()) + assertNull(decoder.readString()) + assertNull(decoder.readEnum()) + + } + + @Test + fun testMissingFlush() { + val fieldNr = 17 + val buffer = Buffer() + + val encoder = WireEncoder(buffer) + encoder.writeBool(fieldNr, true) + // Intentionally not calling flush() + + // The data is not being written to the buffer yet + val decoder = WireDecoder(buffer) + assertNull(decoder.readTag()) + decoder.close() + } + + @Test + fun testMultipleFieldsEncodeDecode() { + val buffer = Buffer() + val encoder = WireEncoder(buffer) + + // Write multiple fields of different types + assertTrue(encoder.writeBool(1, true)) + assertTrue(encoder.writeInt32(2, 42)) + assertTrue(encoder.writeString(3, "Hello")) + assertTrue(encoder.writeFixed64(4, 123456789uL)) + encoder.flush() + + val decoder = WireDecoder(buffer) + + // Read and verify each field + val tag1 = decoder.readTag() + assertNotNull(tag1) + assertEquals(1, tag1.fieldNr) + assertEquals(WireType.VARINT, tag1.wireType) + val bool = decoder.readBool() + assertNotNull(bool) + assertTrue(bool) + + val tag2 = decoder.readTag() + assertNotNull(tag2) + assertEquals(2, tag2.fieldNr) + assertEquals(WireType.VARINT, tag2.wireType) + val int32 = decoder.readInt32() + assertNotNull(int32) + assertEquals(42, int32) + + val tag3 = decoder.readTag() + assertNotNull(tag3) + assertEquals(3, tag3.fieldNr) + assertEquals(WireType.LENGTH_DELIMITED, tag3.wireType) + val string = decoder.readString() + assertNotNull(string) + assertEquals("Hello", string) + + val tag4 = decoder.readTag() + assertNotNull(tag4) + assertEquals(4, tag4.fieldNr) + assertEquals(WireType.FIXED64, tag4.wireType) + val fixed64 = decoder.readFixed64() + assertNotNull(fixed64) + assertEquals(123456789uL, fixed64) + + // No more tags + assertNull(decoder.readTag()) + + decoder.close() + assertTrue(buffer.exhausted()) + } + + @Test + fun testReadAfterClose() { + val fieldNr = 19 + val buffer = Buffer() + + val encoder = WireEncoder(buffer) + assertTrue(encoder.writeBool(fieldNr, true)) + encoder.flush() + + val decoder = WireDecoder(buffer) + decoder.close() + + // Reading after close should either return null or throw an exception + try { + val tag = decoder.readTag() + assertNull(tag) + } catch (e: Exception) { + // Expected exception in some implementations + } + } + + @Test + fun testWriteAfterFlush() { + val buffer = Buffer() + + val encoder = WireEncoder(buffer) + assertTrue(encoder.writeBool(1, true)) + encoder.flush() + + // Writing after flush should still work + assertTrue(encoder.writeInt32(2, 42)) + encoder.flush() + + val decoder = WireDecoder(buffer) + + // Verify both values were written + val tag1 = decoder.readTag() + assertNotNull(tag1) + assertEquals(1, tag1.fieldNr) + val bool = decoder.readBool() + assertNotNull(bool) + assertTrue(bool) + + val tag2 = decoder.readTag() + assertNotNull(tag2) + assertEquals(2, tag2.fieldNr) + val int32 = decoder.readInt32() + assertNotNull(int32) + assertEquals(42, int32) + + decoder.close() + assertTrue(buffer.exhausted()) + } + + @Test + fun testUnicodeStringEncodeDecode() { + val fieldNr = 20 + val testValue = "Hello, 世界! 😊" + val buffer = Buffer() + + val encoder = WireEncoder(buffer) + assertTrue(encoder.writeString(fieldNr, testValue)) + encoder.flush() + + val decoder = WireDecoder(buffer) + + val tag = decoder.readTag() + assertNotNull(tag) + assertEquals(WireType.LENGTH_DELIMITED, tag.wireType) + assertEquals(fieldNr, tag.fieldNr) + + val value = decoder.readString() + assertNotNull(value) + assertEquals(testValue, value) + + decoder.close() + assertTrue(buffer.exhausted()) + } + + @Test + fun testBufferNotExhausted() { + val fieldNr = 1 + val buffer = Buffer() + + val encoder = WireEncoder(buffer) + assertTrue(encoder.writeBool(fieldNr, true)) + assertTrue(encoder.writeBool(fieldNr + 1, true)) + encoder.flush() + + WireDecoder(buffer).use { decoder -> + decoder.readTag() + assertNotNull(decoder.readString()) + } + assertFalse(buffer.exhausted()) + } + + @Test + fun testBufferUsedByMultipleDecoders() { + val buffer = Buffer() + + val field1Nr = 1 + val field2Nr = 2 + val field1Str = "a".repeat(1000000) + val field2Str = "b".repeat(1000000) + + val encoder = WireEncoder(buffer) + assertTrue(encoder.writeString(field1Nr, field1Str)) + assertTrue(encoder.writeString(field2Nr, field2Str)) + encoder.flush() + + WireDecoder(buffer).use { decoder -> + val tag = decoder.readTag() + assertEquals(field1Nr, tag?.fieldNr) + assertEquals(field1Str, decoder.readString()) + } + assertFalse(buffer.exhausted()) + + WireDecoder(buffer).use { decoder -> + val tag = decoder.readTag() + assertEquals(field2Nr, tag?.fieldNr) + assertEquals(field2Str, decoder.readString()) + } + assertTrue(buffer.exhausted()) + } +} diff --git a/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/ZeroCopyInputSourceTest.kt b/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/ZeroCopyInputSourceTest.kt index 1c5af05ee..234d0941a 100644 --- a/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/ZeroCopyInputSourceTest.kt +++ b/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/ZeroCopyInputSourceTest.kt @@ -19,51 +19,299 @@ import platform.posix.memcpy import kotlin.experimental.ExperimentalNativeApi import kotlin.test.Test import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertFalse +import kotlin.test.assertNotEquals +import kotlin.test.assertTrue +import kotlin.test.fail @OptIn(ExperimentalForeignApi::class, ExperimentalNativeApi::class) class ZeroCopyInputSourceTest { + @Test + fun testEmptyBuffer() { + val buffer = Buffer() + val source = ZeroCopyInputSource(buffer) + + // next() should return false for empty buffer + memScoped { + val data = alloc>() + val size = alloc() + assertFalse(source.next(data.ptr, size.ptr)) + } + + // byteCount should be 0 + assertEquals(0, source.byteCount()) + + // skip should return false for empty buffer + assertFalse(source.skip(10)) + + // close should work without errors + source.close() + } @Test - fun simpleTest() { + fun testNextMethod() { + val buffer = Buffer() + val testData = ByteArray(100) { it.toByte() } + buffer.write(testData) + + val source = ZeroCopyInputSource(buffer) + + // First next() call should return true and provide data + val firstRead = source.nextIntoArray() + assertEquals(100, firstRead.size) + assertTrue(firstRead.contentEquals(testData)) + // Second next() call should return false (buffer exhausted) + val secondRead = source.nextIntoArray() + assertEquals(0, secondRead.size) + + // byteCount should reflect the bytes read + assertEquals(100, source.byteCount()) + + source.close() + assertTrue(buffer.exhausted()) + } + @Test + fun testBackUpMethod() { val buffer = Buffer() - fillWithChunks(buffer, 1000, 10) + val testData = ByteArray(100) { it.toByte() } + buffer.write(testData) + + val source = ZeroCopyInputSource(buffer) + + // Read all data + val firstRead = source.nextIntoArray() + assertEquals(100, firstRead.size) - val zeroCopyInputSource = ZeroCopyInputSource(buffer) + // Back up 20 bytes + source.backUp(20) - var i = 0 - var count = 0 + // byteCount should be reduced by backup amount + assertEquals(80, source.byteCount()) + + // Next read should return the backed-up bytes + val secondRead = source.nextIntoArray() + assertEquals(20, secondRead.size) + + // Verify the backed-up bytes are correct (last 20 bytes of original data) + for (i in 0 until 20) { + assertEquals(testData[80 + i], secondRead[i]) + } + + // Buffer should be exhausted now + val thirdRead = source.nextIntoArray() + assertEquals(0, thirdRead.size) + + source.close() + assertTrue(buffer.exhausted()) + } + + @Test + fun testInvalidBackUpSequence() { + val buffer = Buffer() + buffer.write(ByteArray(10)) + val source = ZeroCopyInputSource(buffer) + + // Calling backUp() without a preceding next() should throw + assertFailsWith { + source.backUp(5) + } + + source.close() + } + + @Test + fun testSkipMethod() { + val buffer = Buffer() + val testData = ByteArray(100) { it.toByte() } + buffer.write(testData) + + val source = ZeroCopyInputSource(buffer) + + // Skip 30 bytes + assertTrue(source.skip(30)) + assertEquals(30, source.byteCount()) + + // Reading all left segments + val allLeftBytes = assertNBytesLeft(source, 70) + + // Verify we're reading from the correct position + for (i in 0 until 70) { + assertEquals(testData[30 + i], allLeftBytes.readByte()) + } + + assertEquals(100, source.byteCount()) + + // Skip beyond the end should return false + assertFalse(source.skip(10)) + + source.close() + } + + @Test + fun testMultipleSegments() { + val buffer = Buffer() + // Create multiple segments by writing small chunks + for (i in 0 until 100) { + buffer.write(ByteArray(100) { (i * 100 + it).toByte() }) + } + + val source = ZeroCopyInputSource(buffer) + val allData = mutableListOf() + + // Read all segments + var segmentCount = 0 while (true) { - val read = zeroCopyInputSource.nextIntoArray() - println("$i reads: ${read.size}") - if (read.isEmpty()) { - break - } - if (read.size >= 10) { - val toBackup = (read.size - 1) % ((i + 1) * 100) - count -= toBackup - zeroCopyInputSource.backUp( toBackup) + val segment = source.nextIntoArray() + if (segment.isEmpty()) break + segmentCount++ + allData.addAll(segment.toList()) + } + + // assert there were more than 1 segment; otherwise the test is useless + assertTrue(segmentCount > 1) + + // Verify we read all 100 bytes + assertEquals(100 * 100, allData.size) + assertEquals(100 * 100, source.byteCount()) + + // Verify the data is correct + for (i in 0 until 100 * 100) { + assertEquals(i.toByte(), allData[i]) + } + + source.close() + } + + @Test + fun testCloseMethod() { + val buffer = Buffer() + val testData = ByteArray(100) { it.toByte() } + buffer.write(testData) + + val source = ZeroCopyInputSource(buffer) + + // Read the data from source + source.nextIntoArray() + // Back up 20 bytes which have to be available in the original buffer after closing + source.backUp(20) + + // Close the source + source.close() + + // After closing, the buffer should be valid for reading + assertFalse(buffer.exhausted()) + assertEquals(20, buffer.size) + + // Original buffer should contain last 20 bytes of test data + for (i in 0 until 20) { + assertEquals(testData[80 + i], buffer.readByte()) + } + + + // But the source should not be usable + assertFailsWith(message = "ZeroCopyInputSource has already been closed.") { + memScoped { + val data = alloc>() + val size = alloc() + source.next(data.ptr, size.ptr) + fail("Should not be able to use ZeroCopyInputSource after closing") } - count += read.size - i++ } + } + + @Test + fun testOutOfBoundsBackup() { + val buffer = Buffer() + val testData = ByteArray(100) { it.toByte() } + buffer.write(testData) + + val source = ZeroCopyInputSource(buffer) + + // Read all data + val read = source.nextIntoArray() + assertEquals(100, read.size) + + // Try to back up more bytes than we read + assertFailsWith(message = + "backUp() must not be called more than the number of bytes that were read in next()" + ) { source.backUp(200) } + + source.close() + } + + @Test + fun testMultiChunkConsistency() { + val buffer = Buffer() + fillWithChunks(buffer, 1000, 10) + val total = 1000L * 10 + + val source = ZeroCopyInputSource(buffer) + val seg1 = source.nextIntoArray() + + assertEquals(seg1.size.toLong(), source.byteCount()) + + source.close() + + assertEquals(total - seg1.size, buffer.size) + } + + @Test + fun testMultiChunkBackupConsistency() { + val buffer = Buffer() + fillWithChunks(buffer, 1000, 10) + val total = 1000L * 10 + + val source = ZeroCopyInputSource(buffer) + val seg1 = source.nextIntoArray() + + source.backUp(100) + assertEquals(seg1.size.toLong() - 100, source.byteCount()) - assertEquals(10000, zeroCopyInputSource.byteCount()) - assertEquals(10000, count) + source.close() + + assertEquals(total - seg1.size + 100, buffer.size) + } + + @Test + fun testMultiChunkBackup() { + val buffer = Buffer() + fillWithChunks(buffer, 1000, 10) + val total = 1000L * 10 + + val source = ZeroCopyInputSource(buffer) + val seg1 = source.nextIntoArray() + + assertEquals(source.byteCount(), seg1.size.toLong()) + + source.close() + + assertEquals(total - seg1.size, buffer.size) } private fun fillWithChunks(buffer: Buffer, numberOfChunks: Int, chunkSize: Int) { repeat(numberOfChunks) { i -> buffer.write(ByteArray(chunkSize) { i.toByte() }) } - } } +private fun assertNBytesLeft(source: ZeroCopyInputSource, n: Long): Buffer { + // Reading all left segments + val combined = Buffer() + while (combined.size < n) { + val read = source.nextIntoArray() + assertNotEquals(0, read.size) + combined.write(read) + } + assertEquals(n, combined.size) + return combined +} @OptIn(ExperimentalForeignApi::class) private fun ZeroCopyInputSource.nextIntoArray(): ByteArray = memScoped { diff --git a/grpc/grpcpp-c/include/protowire.h b/grpc/grpcpp-c/include/protowire.h index 5657bb0c8..59e1fbbc2 100644 --- a/grpc/grpcpp-c/include/protowire.h +++ b/grpc/grpcpp-c/include/protowire.h @@ -8,6 +8,9 @@ extern "C" { #endif + //// STD::STRING WRAPPER //// + + // A std::string wrapper that helps reduce copies when C++ api returns std::strings typedef struct pw_string pw_string_t; pw_string_t * pw_string_new(const char *str); diff --git a/grpc/grpcpp-c/src/protowire.cpp b/grpc/grpcpp-c/src/protowire.cpp index 0f2eeab08..d6f553bf6 100644 --- a/grpc/grpcpp-c/src/protowire.cpp +++ b/grpc/grpcpp-c/src/protowire.cpp @@ -37,7 +37,8 @@ class SourceStream final : public pb::io::ZeroCopyInputStream { } bool Next(const void **data, int *size) override { - return input.next(input.ctx, data, size); + auto result = input.next(input.ctx, data, size); + return result; }; void BackUp(int count) override { @@ -78,7 +79,7 @@ struct pw_decoder { protowire::SourceStream ss; pb::io::CodedInputStream cis; - pw_decoder(pw_zero_copy_input_t input) + explicit pw_decoder(pw_zero_copy_input_t input) : ss(input), cis(&ss) {} }; @@ -112,9 +113,9 @@ extern "C" { return !self->cos.HadError(); } - // check if there was an error + // check that there was no error static bool check(pw_encoder_t *self) { - return self->cos.HadError(); + return !self->cos.HadError(); } #define WRITE_FIELD_FUNC( funcSuffix, wireTy, cTy) \ From 7bc2e218e8f8b0d746652eec5bf6cd8c09e0db4c Mon Sep 17 00:00:00 2001 From: Johannes Zottele Date: Wed, 23 Jul 2025 16:10:13 +0200 Subject: [PATCH 06/16] grpc-native: Add C++ documentation Signed-off-by: Johannes Zottele --- .../rpc/grpc/internal/WireCodecTest.kt | 2 +- grpc/grpcpp-c/include/protowire.h | 25 ++++- grpc/grpcpp-c/src/protowire.cpp | 101 +++++++++++------- 3 files changed, 86 insertions(+), 42 deletions(-) diff --git a/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/WireCodecTest.kt b/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/WireCodecTest.kt index c7651ae24..033bef9b3 100644 --- a/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/WireCodecTest.kt +++ b/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/WireCodecTest.kt @@ -14,7 +14,7 @@ import kotlin.test.assertNotNull import kotlin.test.assertNull import kotlin.test.assertTrue -// TODO: Move this into the commonTest +// TODO: Move this to the commonTest @OptIn(ExperimentalForeignApi::class, ExperimentalNativeApi::class) class WireCodecTest { diff --git a/grpc/grpcpp-c/include/protowire.h b/grpc/grpcpp-c/include/protowire.h index 59e1fbbc2..822a4f1b8 100644 --- a/grpc/grpcpp-c/include/protowire.h +++ b/grpc/grpcpp-c/include/protowire.h @@ -4,6 +4,8 @@ #include #include +// Defines the C wrapper around the C++ Wire Format encoding/decoding API +// (WireFormatLite, Coded(Input|Output)Stream, ZeroCopyInputStream, CopingOutputStream) #ifdef __cplusplus extern "C" { #endif @@ -22,7 +24,13 @@ extern "C" { typedef struct pw_encoder pw_encoder_t; - pw_encoder_t *pw_encoder_new(void* ctx, bool (*)(void* ctx, const void* buf, int size)); + /** + * Create a new pw_encoder_t that wraps a CodedOutputStream to encode values into a wire format stream. + * + * @param ctx a stable pointer to a Kotlin managed object, used by the K/N sink callback to access Kotlin objects. + * @param sink_fn the K/N callback function to write encoded data into the kotlinx.io.Sink. + */ + pw_encoder_t *pw_encoder_new(void* ctx, bool (*sink_fn)(void* ctx, const void* buf, int size)); void pw_encoder_delete(pw_encoder_t *self); bool pw_encoder_flush(pw_encoder_t *self); @@ -44,11 +52,16 @@ extern "C" { bool pw_encoder_write_bytes(pw_encoder_t *self, int field_no, pw_string_t *value); - //// WIRE DECODER //// typedef struct pw_decoder pw_decoder_t; + /** + * Holds callbacks corresponding to the methods of a ZeroCopyInputStream. + * They are called to retrieve data from the K/N side with a minimal number of copies. + * + * For method documentation see the ZeroCopyInputStream (C++) interface and the ZeroCopyInputSource (Kotlin) class. + */ typedef struct pw_zero_copy_input { void *ctx; bool (*next)(void *ctx, const void **data, int *size); @@ -57,13 +70,15 @@ extern "C" { int64_t (*byteCount)(void *ctx); } pw_zero_copy_input_t; + /** + * Create a new pw_decoder_t that wraps a CodedInputStream to decode values from a wire format stream. + * + * @param zero_copy_input holds callbacks to the K/N side, matching the ZeroCopyInputStream interface. + */ pw_decoder_t * pw_decoder_new(pw_zero_copy_input_t zero_copy_input); void pw_decoder_delete(pw_decoder_t *self); uint32_t pw_decoder_read_tag(pw_decoder_t *self); - - /** Read primitive **/ - bool pw_decoder_read_bool(pw_decoder_t *self, bool *value); bool pw_decoder_read_int32(pw_decoder_t *self, int32_t *value); bool pw_decoder_read_int64(pw_decoder_t *self, int64_t *value); diff --git a/grpc/grpcpp-c/src/protowire.cpp b/grpc/grpcpp-c/src/protowire.cpp index d6f553bf6..c97b829d4 100644 --- a/grpc/grpcpp-c/src/protowire.cpp +++ b/grpc/grpcpp-c/src/protowire.cpp @@ -14,48 +14,78 @@ namespace pb = google::protobuf; typedef pb::internal::WireFormatLite WireFormatLite; namespace protowire { -class SinkStream final : public pb::io::CopyingOutputStream { -public: - SinkStream(void *ctx, bool(*sink)(void *ctx, const void *buffer, int size)) - : ctx(ctx), - sink(sink) { - } + /** + * A bridge that passes write calls to the K/N side, to write the data into a kotlinx.io.Sink. + * + * This reduces the amount of copying, as the callback on the K/N may directly use the + * buffer pointer to copy the whole chunk at into the stream. + */ + class SinkStream final : public pb::io::CopyingOutputStream { + public: + /** + * Constructs the stream with a ctx pointer and a callback to the K/N side. + * The ctx pointer is used to on the K/N to reference a Kotlin managed object + * from within its static callback function. + * + * @param ctx the context used by the K/N side to reference Kotlin managed objects. + * @param sink the K/N callback to write data into the sink + */ + SinkStream(void *ctx, bool(*sink)(void *ctx, const void *buffer, int size)) + : ctx(ctx), + sink(sink) { + } - bool Write(const void *buffer, int size) override { - return sink(ctx, buffer, size); - } + bool Write(const void *buffer, int size) override { + return sink(ctx, buffer, size); + } -private: - void *ctx; - bool (*sink)(void *ctx, const void *buffer, int size); -}; + private: + void *ctx; + bool (*sink)(void *ctx, const void *buffer, int size); + }; -class SourceStream final : public pb::io::ZeroCopyInputStream { -public: - explicit SourceStream(const pw_zero_copy_input_t &input) - : input(input) { - } + /** + * A bridge that passes read calls to the K/N side, to read data from a kotlinx.io.Buffer. + * + * This allows efficient data reading from the K/N side buffer, as it allows + * directly accessing continuous memory blocks from within the buffer, instead of copying them + * via C-Interop. + * + * All ZeroCopyInputStream methods are delegated to the K/N call back functions, hold in + * the pw_zero_copy_input_t. + */ + class BufferSourceStream final : public pb::io::ZeroCopyInputStream { + public: + /** + * Constructs the BufferSourceStream to access kotlinx.io.Buffer segments directly, without + * copying them via C-Interop. + * + * @param input a struct containing K/N callbacks for all methods of the ZeroCopyInputStream. + */ + explicit BufferSourceStream(const pw_zero_copy_input_t &input) + : input(input) { + } - bool Next(const void **data, int *size) override { - auto result = input.next(input.ctx, data, size); - return result; - }; + bool Next(const void **data, int *size) override { + auto result = input.next(input.ctx, data, size); + return result; + }; - void BackUp(int count) override { - return input.backUp(input.ctx, count); - }; + void BackUp(int count) override { + return input.backUp(input.ctx, count); + }; - bool Skip(int count) override { - return input.skip(input.ctx, count); - }; + bool Skip(int count) override { + return input.skip(input.ctx, count); + }; - int64_t ByteCount() const override { - return input.byteCount(input.ctx); - }; + int64_t ByteCount() const override { + return input.byteCount(input.ctx); + }; -private: - pw_zero_copy_input_t input; -}; + private: + pw_zero_copy_input_t input; + }; } @@ -72,11 +102,10 @@ struct pw_encoder { : sinkStream(std::move(sink)), cosa(&sinkStream), cos(&cosa) {} - }; struct pw_decoder { - protowire::SourceStream ss; + protowire::BufferSourceStream ss; pb::io::CodedInputStream cis; explicit pw_decoder(pw_zero_copy_input_t input) From 8be7b5265b2781b1df58b0ca8b5130e1cdd32f47 Mon Sep 17 00:00:00 2001 From: Johannes Zottele Date: Wed, 23 Jul 2025 17:38:15 +0200 Subject: [PATCH 07/16] grpc-native: Optimize String encoding by avoiding converting Kotlin string to std::string Signed-off-by: Johannes Zottele --- .../nativeInterop/cinterop/libprotowire.def | 2 ++ .../rpc/grpc/internal/WireEncoder.native.kt | 12 +++++++---- .../rpc/grpc/internal/WireCodecTest.kt | 20 +++++++++++++++++++ grpc/grpcpp-c/include/protowire.h | 3 ++- grpc/grpcpp-c/src/protowire.cpp | 10 +++++++--- 5 files changed, 39 insertions(+), 8 deletions(-) diff --git a/grpc/grpc-core/src/nativeInterop/cinterop/libprotowire.def b/grpc/grpc-core/src/nativeInterop/cinterop/libprotowire.def index 6cf4a33d5..3868c5fc6 100644 --- a/grpc/grpc-core/src/nativeInterop/cinterop/libprotowire.def +++ b/grpc/grpc-core/src/nativeInterop/cinterop/libprotowire.def @@ -1,4 +1,6 @@ headers = protowire.h headerFilter = protowire.h +noStringConversion = pw_encoder_write_string + staticLibraries = libprotowire_static.a \ No newline at end of file diff --git a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.native.kt b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.native.kt index 1ac8086cb..86438d447 100644 --- a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.native.kt +++ b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.native.kt @@ -11,6 +11,8 @@ import kotlin.experimental.ExperimentalNativeApi import kotlin.native.ref.createCleaner +// TODO: Evaluate if we should implement a ZeroCopyOutputSink (similar to the ZeroCopyInputSource) +// to reduce the number of copies during encoding. @OptIn(ExperimentalForeignApi::class, ExperimentalNativeApi::class) internal class WireEncoderNative(private val sink: Sink): WireEncoder { /** @@ -94,10 +96,12 @@ internal class WireEncoderNative(private val sink: Sink): WireEncoder { } override fun writeString(fieldNr: Int, value: String): Boolean { - val str = pw_string_new(value) ?: error("Failed to create string") - val result = pw_encoder_write_string(raw, fieldNr, str) - pw_string_delete(str) - return result; + if (value.isEmpty()) { + return pw_encoder_write_string(raw, fieldNr, null, 0) + } + return value.usePinned { + pw_encoder_write_string(raw, fieldNr, it.addressOf(0).reinterpret(), value.length) + } } override fun flush() { diff --git a/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/WireCodecTest.kt b/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/WireCodecTest.kt index 033bef9b3..0fb20b5f2 100644 --- a/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/WireCodecTest.kt +++ b/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/WireCodecTest.kt @@ -553,4 +553,24 @@ class WireCodecTest { } assertTrue(buffer.exhausted()) } + + @Test + fun testEmptyString() { + val buffer = Buffer() + + val encoder = WireEncoder(buffer) + assertTrue(encoder.writeString(1, "")) + encoder.flush() + + val decoder = WireDecoder(buffer) + + val tag = decoder.readTag() + assertNotNull(tag) + assertEquals(1, tag.fieldNr) + assertEquals(WireType.LENGTH_DELIMITED, tag.wireType) + + val str = decoder.readString() + assertNotNull(str) + assertEquals("", str) + } } diff --git a/grpc/grpcpp-c/include/protowire.h b/grpc/grpcpp-c/include/protowire.h index 822a4f1b8..9449ee359 100644 --- a/grpc/grpcpp-c/include/protowire.h +++ b/grpc/grpcpp-c/include/protowire.h @@ -48,7 +48,7 @@ extern "C" { bool pw_encoder_write_float(pw_encoder_t *self, int field_no, float value); bool pw_encoder_write_double(pw_encoder_t *self, int field_no, double value); bool pw_encoder_write_enum(pw_encoder_t *self, int field_no, int value); - bool pw_encoder_write_string(pw_encoder_t *self, int field_no, pw_string_t *value); + bool pw_encoder_write_string(pw_encoder_t *self, int field_no, const char *data, int size); bool pw_encoder_write_bytes(pw_encoder_t *self, int field_no, pw_string_t *value); @@ -70,6 +70,7 @@ extern "C" { int64_t (*byteCount)(void *ctx); } pw_zero_copy_input_t; + /** * Create a new pw_decoder_t that wraps a CodedInputStream to decode values from a wire format stream. * diff --git a/grpc/grpcpp-c/src/protowire.cpp b/grpc/grpcpp-c/src/protowire.cpp index c97b829d4..d8a8554c0 100644 --- a/grpc/grpcpp-c/src/protowire.cpp +++ b/grpc/grpcpp-c/src/protowire.cpp @@ -101,7 +101,9 @@ struct pw_encoder { explicit pw_encoder(protowire::SinkStream sink) : sinkStream(std::move(sink)), cosa(&sinkStream), - cos(&cosa) {} + cos(&cosa) { + cos.EnableAliasing(true); + } }; struct pw_decoder { @@ -166,8 +168,10 @@ extern "C" { WRITE_FIELD_FUNC( sfixed64, SFixed64, int64_t) WRITE_FIELD_FUNC( enum, Enum, int) - bool pw_encoder_write_string(pw_encoder_t *self, int field_no, pw_string_t *value) { - WireFormatLite::WriteString(field_no, value->str, &self->cos); + bool pw_encoder_write_string(pw_encoder_t *self, int field_no, const char *data, int size) { + WireFormatLite::WriteTag(field_no, WireFormatLite::WIRETYPE_LENGTH_DELIMITED, &self->cos); + self->cos.WriteVarint32(size); + self->cos.WriteRawMaybeAliased(data, size); return check(self); } bool pw_encoder_write_bytes(pw_encoder_t *self, int field_no, pw_string_t *value) { From 44d226e14388333a02b094e85c02b802e5160da1 Mon Sep 17 00:00:00 2001 From: Johannes Zottele Date: Wed, 23 Jul 2025 18:30:35 +0200 Subject: [PATCH 08/16] grpc-native: Implement encoding/decoding of bytes Signed-off-by: Johannes Zottele --- .../kotlinx/rpc/grpc/internal/WireDecoder.kt | 1 + .../kotlinx/rpc/grpc/internal/WireEncoder.kt | 1 + .../rpc/grpc/internal/WireDecoder.native.kt | 11 ++++++ .../rpc/grpc/internal/WireEncoder.native.kt | 10 +++++ .../rpc/grpc/internal/WireCodecTest.kt | 37 +++++++++++++++++++ grpc/grpcpp-c/include/protowire.h | 4 +- grpc/grpcpp-c/src/protowire.cpp | 12 +++--- 7 files changed, 70 insertions(+), 6 deletions(-) diff --git a/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.kt b/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.kt index 3719c4058..2930b2858 100644 --- a/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.kt +++ b/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.kt @@ -43,6 +43,7 @@ internal interface WireDecoder: AutoCloseable { fun readSFixed64(): Long? fun readEnum(): Int? fun readString(): String? + fun readBytes(): ByteArray? } /** diff --git a/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.kt b/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.kt index f13aa4f86..92401b496 100644 --- a/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.kt +++ b/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.kt @@ -29,6 +29,7 @@ internal interface WireEncoder { fun writeEnum(fieldNr: Int, value: Int): Boolean fun writeString(fieldNr: Int, value: String): Boolean fun flush() + fun writeBytes(fieldNr: Int, value: ByteArray): Boolean } diff --git a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.native.kt b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.native.kt index 3aba74cb4..cdf9ae93d 100644 --- a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.native.kt +++ b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.native.kt @@ -152,6 +152,7 @@ internal class WireDecoderNative(private val source: Buffer): WireDecoder { return null } + // TODO: Is it possible to avoid copying the c_str, by directly allocating a K/N String (as in readBytes)? override fun readString(): String? = memScoped { val str = alloc>() val ok = pw_decoder_read_string(raw, str.ptr) @@ -162,6 +163,16 @@ internal class WireDecoderNative(private val source: Buffer): WireDecoder { pw_string_delete(str.value) } } + + override fun readBytes(): ByteArray? { + val length = readInt32() ?: return null + if (length == 0) return ByteArray(0) + val bytes = ByteArray(length) + bytes.usePinned { + pw_decoder_read_raw_bytes(raw, it.addressOf(0), length) + } + return bytes + } } internal actual fun WireDecoder(source: Buffer): WireDecoder = WireDecoderNative(source) \ No newline at end of file diff --git a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.native.kt b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.native.kt index 86438d447..85c6f1713 100644 --- a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.native.kt +++ b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.native.kt @@ -9,6 +9,7 @@ import kotlinx.io.Sink import libprotowire.* import kotlin.experimental.ExperimentalNativeApi import kotlin.native.ref.createCleaner +import kotlin.text.isEmpty // TODO: Evaluate if we should implement a ZeroCopyOutputSink (similar to the ZeroCopyInputSource) @@ -104,6 +105,15 @@ internal class WireEncoderNative(private val sink: Sink): WireEncoder { } } + override fun writeBytes(fieldNr: Int, value: ByteArray): Boolean { + if (value.isEmpty()) { + return pw_encoder_write_bytes(raw, fieldNr, null, 0) + } + return value.usePinned { + pw_encoder_write_bytes(raw, fieldNr, it.addressOf(0), value.size) + } + } + override fun flush() { pw_encoder_flush(raw) } diff --git a/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/WireCodecTest.kt b/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/WireCodecTest.kt index 0fb20b5f2..278e350a1 100644 --- a/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/WireCodecTest.kt +++ b/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/WireCodecTest.kt @@ -573,4 +573,41 @@ class WireCodecTest { assertNotNull(str) assertEquals("", str) } + + @Test + fun testEmptyByteArray() { + val buffer = Buffer() + + val encoder = WireEncoder(buffer) + assertTrue(encoder.writeBytes(1, ByteArray(0))) + encoder.flush() + + val decoder = WireDecoder(buffer) + + val tag = decoder.readTag() + assertNotNull(tag) + assertEquals(1, tag.fieldNr) + assertEquals(WireType.LENGTH_DELIMITED, tag.wireType) + + val bytes = decoder.readBytes() + assertNotNull(bytes) + assertEquals(0, bytes.size) + } + + @Test + fun testBytesEncodeDecode() { + val buffer = Buffer() + val encoder = WireEncoder(buffer) + + val bytes = ByteArray(1000000) { it.toByte() } + + assertTrue(encoder.writeBytes(1, bytes)) + encoder.flush() + + val decoder = WireDecoder(buffer) + val tag = decoder.readTag() + assertNotNull(tag) + assertEquals(1, tag.fieldNr) + assertEquals(WireType.LENGTH_DELIMITED, tag.wireType) + } } diff --git a/grpc/grpcpp-c/include/protowire.h b/grpc/grpcpp-c/include/protowire.h index 9449ee359..2fe890b83 100644 --- a/grpc/grpcpp-c/include/protowire.h +++ b/grpc/grpcpp-c/include/protowire.h @@ -49,7 +49,7 @@ extern "C" { bool pw_encoder_write_double(pw_encoder_t *self, int field_no, double value); bool pw_encoder_write_enum(pw_encoder_t *self, int field_no, int value); bool pw_encoder_write_string(pw_encoder_t *self, int field_no, const char *data, int size); - bool pw_encoder_write_bytes(pw_encoder_t *self, int field_no, pw_string_t *value); + bool pw_encoder_write_bytes(pw_encoder_t *self, int field_no, const void *data, int size); //// WIRE DECODER //// @@ -95,6 +95,8 @@ extern "C" { bool pw_decoder_read_double(pw_decoder_t *self, double *value); bool pw_decoder_read_enum(pw_decoder_t *self, int *value); bool pw_decoder_read_string(pw_decoder_t *self, pw_string_t **opaque_string); + // To read an actual bytes field, you must combine read_int32 and this function + bool pw_decoder_read_raw_bytes(pw_decoder_t *self, void* buffer, int size); #ifdef __cplusplus } diff --git a/grpc/grpcpp-c/src/protowire.cpp b/grpc/grpcpp-c/src/protowire.cpp index d8a8554c0..96b71e2f9 100644 --- a/grpc/grpcpp-c/src/protowire.cpp +++ b/grpc/grpcpp-c/src/protowire.cpp @@ -169,16 +169,14 @@ extern "C" { WRITE_FIELD_FUNC( enum, Enum, int) bool pw_encoder_write_string(pw_encoder_t *self, int field_no, const char *data, int size) { + return pw_encoder_write_bytes(self, field_no, data, size); + } + bool pw_encoder_write_bytes(pw_encoder_t *self, int field_no, const void *data, int size) { WireFormatLite::WriteTag(field_no, WireFormatLite::WIRETYPE_LENGTH_DELIMITED, &self->cos); self->cos.WriteVarint32(size); self->cos.WriteRawMaybeAliased(data, size); return check(self); } - bool pw_encoder_write_bytes(pw_encoder_t *self, int field_no, pw_string_t *value) { - WireFormatLite::WriteBytes(field_no, value->str, &self->cos); - return check(self); - } - pw_decoder_t *pw_decoder_new(pw_zero_copy_input_t zero_copy_input) { return new pw_decoder_t(zero_copy_input); @@ -214,4 +212,8 @@ extern "C" { *string_ref = new pw_string_t; return WireFormatLite::ReadString(&self->cis, &(*string_ref)->str); } + + bool pw_decoder_read_raw_bytes(pw_decoder_t *self, void* buffer, int size) { + return self->cis.ReadRaw(buffer, size); + } } From 47e7eff8d07c83de8cc5e0740e7d1395cb5da36e Mon Sep 17 00:00:00 2001 From: Johannes Zottele Date: Thu, 24 Jul 2025 11:47:56 +0200 Subject: [PATCH 09/16] grpc-native: Add float and double encode/decode Signed-off-by: Johannes Zottele --- .../kotlinx/rpc/grpc/internal/WireDecoder.kt | 2 + .../kotlinx/rpc/grpc/internal/WireEncoder.kt | 2 + .../rpc/grpc/internal/WireDecoder.native.kt | 21 +++++++- .../rpc/grpc/internal/WireEncoder.native.kt | 8 +++ .../rpc/grpc/internal/WireCodecTest.kt | 50 +++++++++++++++++++ grpc/grpcpp-c/include/protowire.h | 1 + grpc/grpcpp-c/src/protowire.cpp | 4 ++ 7 files changed, 87 insertions(+), 1 deletion(-) diff --git a/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.kt b/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.kt index 2930b2858..41a14a527 100644 --- a/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.kt +++ b/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.kt @@ -41,6 +41,8 @@ internal interface WireDecoder: AutoCloseable { fun readFixed64(): ULong? fun readSFixed32(): Int? fun readSFixed64(): Long? + fun readFloat(): Float? + fun readDouble(): Double? fun readEnum(): Int? fun readString(): String? fun readBytes(): ByteArray? diff --git a/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.kt b/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.kt index 92401b496..7d9b31f1f 100644 --- a/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.kt +++ b/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.kt @@ -26,6 +26,8 @@ internal interface WireEncoder { fun writeFixed64(fieldNr: Int, value: ULong): Boolean fun writeSFixed32(fieldNr: Int, value: Int): Boolean fun writeSFixed64(fieldNr: Int, value: Long): Boolean + fun writeFloat(fieldNr: Int, value: Float): Boolean + fun writeDouble(fieldNr: Int, value: Double): Boolean fun writeEnum(fieldNr: Int, value: Int): Boolean fun writeString(fieldNr: Int, value: String): Boolean fun flush() diff --git a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.native.kt b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.native.kt index cdf9ae93d..e40272ad2 100644 --- a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.native.kt +++ b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.native.kt @@ -10,7 +10,7 @@ import libprotowire.* import kotlin.experimental.ExperimentalNativeApi @OptIn(ExperimentalForeignApi::class, ExperimentalNativeApi::class) -internal class WireDecoderNative(private val source: Buffer): WireDecoder { +internal class WireDecoderNative(source: Buffer): WireDecoder { // wraps the source in a class that allows to pass data from the source buffer to the C++ encoder // without copying it to an intermediate byte array. @@ -144,6 +144,22 @@ internal class WireDecoderNative(private val source: Buffer): WireDecoder { return null } + override fun readFloat(): Float? = memScoped { + val value = alloc() + if (pw_decoder_read_float(raw, value.ptr)) { + return value.value + } + return null + } + + override fun readDouble(): Double? = memScoped { + val value = alloc() + if (pw_decoder_read_double(raw, value.ptr)) { + return value.value + } + return null + } + override fun readEnum(): Int? = memScoped { val value = alloc() if (pw_decoder_read_enum(raw, value.ptr)) { @@ -164,8 +180,11 @@ internal class WireDecoderNative(private val source: Buffer): WireDecoder { } } + // TODO: Should readBytes return a buffer? The current approach is dangerous as one could send a + // huge length (max 2GB) and we would just allocate the array of that length. override fun readBytes(): ByteArray? { val length = readInt32() ?: return null + if (length < 0) return null if (length == 0) return ByteArray(0) val bytes = ByteArray(length) bytes.usePinned { diff --git a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.native.kt b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.native.kt index 85c6f1713..1ba3ba4d1 100644 --- a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.native.kt +++ b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.native.kt @@ -92,6 +92,14 @@ internal class WireEncoderNative(private val sink: Sink): WireEncoder { return pw_encoder_write_sfixed64(raw, fieldNr, value) } + override fun writeFloat(fieldNr: Int, value: Float): Boolean { + return pw_encoder_write_float(raw, fieldNr, value) + } + + override fun writeDouble(fieldNr: Int, value: Double): Boolean { + return pw_encoder_write_double(raw, fieldNr, value) + } + override fun writeEnum(fieldNr: Int, value: Int): Boolean { return pw_encoder_write_enum(raw, fieldNr, value) } diff --git a/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/WireCodecTest.kt b/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/WireCodecTest.kt index 278e350a1..26c61a8d7 100644 --- a/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/WireCodecTest.kt +++ b/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/WireCodecTest.kt @@ -610,4 +610,54 @@ class WireCodecTest { assertEquals(1, tag.fieldNr) assertEquals(WireType.LENGTH_DELIMITED, tag.wireType) } + + @Test + fun testDoubleEncodeDecode() { + val fieldNr = 21 + val testValue = 3.14159265359 + val buffer = Buffer() + + val encoder = WireEncoder(buffer) + assertTrue(encoder.writeDouble(fieldNr, testValue)) + encoder.flush() + + val decoder = WireDecoder(buffer) + + val tag = decoder.readTag() + assertNotNull(tag) + assertEquals(WireType.FIXED64, tag.wireType) + assertEquals(fieldNr, tag.fieldNr) + + val value = decoder.readDouble() + assertNotNull(value) + assertEquals(testValue, value) + + decoder.close() + assertTrue(buffer.exhausted()) + } + + @Test + fun testFloatEncodeDecode() { + val fieldNr = 22 + val testValue = 3.14159f + val buffer = Buffer() + + val encoder = WireEncoder(buffer) + assertTrue(encoder.writeFloat(fieldNr, testValue)) + encoder.flush() + + val decoder = WireDecoder(buffer) + + val tag = decoder.readTag() + assertNotNull(tag) + assertEquals(WireType.FIXED32, tag.wireType) + assertEquals(fieldNr, tag.fieldNr) + + val value = decoder.readFloat() + assertNotNull(value) + assertEquals(testValue, value) + + decoder.close() + assertTrue(buffer.exhausted()) + } } diff --git a/grpc/grpcpp-c/include/protowire.h b/grpc/grpcpp-c/include/protowire.h index 2fe890b83..ddd97fa84 100644 --- a/grpc/grpcpp-c/include/protowire.h +++ b/grpc/grpcpp-c/include/protowire.h @@ -98,6 +98,7 @@ extern "C" { // To read an actual bytes field, you must combine read_int32 and this function bool pw_decoder_read_raw_bytes(pw_decoder_t *self, void* buffer, int size); + #ifdef __cplusplus } #endif diff --git a/grpc/grpcpp-c/src/protowire.cpp b/grpc/grpcpp-c/src/protowire.cpp index 96b71e2f9..eebaa2e02 100644 --- a/grpc/grpcpp-c/src/protowire.cpp +++ b/grpc/grpcpp-c/src/protowire.cpp @@ -166,6 +166,8 @@ extern "C" { WRITE_FIELD_FUNC( fixed64, Fixed64, uint64_t) WRITE_FIELD_FUNC( sfixed32, SFixed32, int32_t) WRITE_FIELD_FUNC( sfixed64, SFixed64, int64_t) + WRITE_FIELD_FUNC( float, Float, float) + WRITE_FIELD_FUNC( double, Double, double) WRITE_FIELD_FUNC( enum, Enum, int) bool pw_encoder_write_string(pw_encoder_t *self, int field_no, const char *data, int size) { @@ -206,6 +208,8 @@ extern "C" { READ_VAL_FUNC( fixed64, FIXED64, uint64_t) READ_VAL_FUNC( sfixed32, SFIXED32, int32_t) READ_VAL_FUNC( sfixed64, SFIXED64, int64_t) + READ_VAL_FUNC( float, FLOAT, float) + READ_VAL_FUNC( double, DOUBLE, double) READ_VAL_FUNC( enum, ENUM, int) bool pw_decoder_read_string(pw_decoder_t *self, pw_string_t **string_ref) { From 34c92446db643b33987b61b03ec5d277c4226612 Mon Sep 17 00:00:00 2001 From: Johannes Zottele Date: Thu, 24 Jul 2025 19:17:45 +0200 Subject: [PATCH 10/16] grpc-native: Add packed Fixed32 encoding and decoding Signed-off-by: Johannes Zottele --- grpc/grpc-core/build.gradle.kts | 6 ++ .../kotlinx/rpc/grpc/internal/WireDecoder.kt | 1 + .../kotlinx/rpc/grpc/internal/WireEncoder.kt | 1 + .../rpc/grpc/internal/WireDecoder.native.kt | 61 +++++++++++++++++-- .../rpc/grpc/internal/WireEncoder.native.kt | 23 +++++-- .../rpc/grpc/internal/WireCodecTest.kt | 38 ++++++++++++ versions-root/libs.versions.toml | 2 + 7 files changed, 122 insertions(+), 10 deletions(-) diff --git a/grpc/grpc-core/build.gradle.kts b/grpc/grpc-core/build.gradle.kts index 243b912e2..d45233654 100644 --- a/grpc/grpc-core/build.gradle.kts +++ b/grpc/grpc-core/build.gradle.kts @@ -61,6 +61,12 @@ kotlin { } } + nativeMain { + dependencies { + implementation(libs.kotlinx.collections.immutable) + } + } + nativeTest { dependencies { implementation(kotlin("test")) diff --git a/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.kt b/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.kt index 41a14a527..cfbf70a7e 100644 --- a/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.kt +++ b/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.kt @@ -46,6 +46,7 @@ internal interface WireDecoder: AutoCloseable { fun readEnum(): Int? fun readString(): String? fun readBytes(): ByteArray? + fun readPackedFixed32(): List? } /** diff --git a/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.kt b/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.kt index 7d9b31f1f..ba82bc399 100644 --- a/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.kt +++ b/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.kt @@ -32,6 +32,7 @@ internal interface WireEncoder { fun writeString(fieldNr: Int, value: String): Boolean fun flush() fun writeBytes(fieldNr: Int, value: ByteArray): Boolean + fun writePackedFixed32(fieldNr: Int, value: UIntArray): Boolean } diff --git a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.native.kt b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.native.kt index e40272ad2..5ed15b2a7 100644 --- a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.native.kt +++ b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.native.kt @@ -5,12 +5,16 @@ package kotlinx.rpc.grpc.internal import kotlinx.cinterop.* +import kotlinx.collections.immutable.persistentListOf import kotlinx.io.Buffer import libprotowire.* import kotlin.experimental.ExperimentalNativeApi +import kotlin.math.min + +private const val MAX_PACKED_BULK_SIZE: Int = 1_000_000 @OptIn(ExperimentalForeignApi::class, ExperimentalNativeApi::class) -internal class WireDecoderNative(source: Buffer): WireDecoder { +internal class WireDecoderNative(private val source: Buffer): WireDecoder { // wraps the source in a class that allows to pass data from the source buffer to the C++ encoder // without copying it to an intermediate byte array. @@ -180,18 +184,67 @@ internal class WireDecoderNative(source: Buffer): WireDecoder { } } - // TODO: Should readBytes return a buffer? The current approach is dangerous as one could send a - // huge length (max 2GB) and we would just allocate the array of that length. + // TODO: Should readBytes return a buffer, to prevent allocation of large contiguous memory blocks ? KRPC-182 override fun readBytes(): ByteArray? { val length = readInt32() ?: return null if (length < 0) return null + // check if the remaining buffer size is less than the set length, + // we can early abort, without allocating unnecessary memory + if (source.size < length) return null if (length == 0) return ByteArray(0) val bytes = ByteArray(length) + var ok = true bytes.usePinned { - pw_decoder_read_raw_bytes(raw, it.addressOf(0), length) + ok = pw_decoder_read_raw_bytes(raw, it.addressOf(0), length) } + if (!ok) return null return bytes } + + /* + * Based on the length of the packed repeated field, one of two list strategies is chosen. + * If the length is less or equal a specific threshold (MAX_PACKED_BULK_SIZE), + * a single array list is filled with the buffer-packed value (two copies). + * Otherwise, a kotlinx.collections.immutable.PersistentList is used to split allocation in several chunks. + * To build the persistent list, a buffer array is allocated that is used for fast copy from C++ to Kotlin. + * + * Note that this implementation assumes a little endian memory order. + */ + override fun readPackedFixed32(): List? { + var byteLen = readInt32() ?: return null + if (byteLen < 0) return null + if (source.size < byteLen) return null + if (byteLen % UInt.SIZE_BYTES != 0 ) return null + val count = byteLen / UInt.SIZE_BYTES + if (byteLen == 0) return emptyList() + + if (count <= MAX_PACKED_BULK_SIZE) { + // this implementation assumes that the program is running on little endian machines. + val arr = UIntArray(count) + arr.usePinned { + pw_decoder_read_raw_bytes(raw, it.addressOf(0), byteLen) + } + return ArrayList(arr) + } else { + val bufByteLen = MAX_PACKED_BULK_SIZE + val bufLen = bufByteLen / UInt.SIZE_BYTES + val buffer = UIntArray(bufLen) + var list = persistentListOf() + buffer.usePinned { + while (byteLen > 0) { + val written = min(bufByteLen, byteLen) + pw_decoder_read_raw_bytes(raw, it.addressOf(0), written) + list = if (written == bufByteLen) { + list.addAll(buffer) + } else { + list.addAll(buffer.copyOfRange(0, written / UInt.SIZE_BYTES)) + } + byteLen -= written + } + } + return list + } + } } internal actual fun WireDecoder(source: Buffer): WireDecoder = WireDecoderNative(source) \ No newline at end of file diff --git a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.native.kt b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.native.kt index 1ba3ba4d1..3e7540cab 100644 --- a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.native.kt +++ b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.native.kt @@ -7,6 +7,7 @@ package kotlinx.rpc.grpc.internal import kotlinx.cinterop.* import kotlinx.io.Sink import libprotowire.* +import kotlin.collections.isEmpty import kotlin.experimental.ExperimentalNativeApi import kotlin.native.ref.createCleaner import kotlin.text.isEmpty @@ -48,6 +49,10 @@ internal class WireEncoderNative(private val sink: Sink): WireEncoder { pw_encoder_delete(it) } + override fun flush() { + pw_encoder_flush(raw) + } + override fun writeBool(field: Int, value: Boolean): Boolean { return pw_encoder_write_bool(raw, field, value) } @@ -104,13 +109,12 @@ internal class WireEncoderNative(private val sink: Sink): WireEncoder { return pw_encoder_write_enum(raw, fieldNr, value) } - override fun writeString(fieldNr: Int, value: String): Boolean { + override fun writeString(fieldNr: Int, value: String): Boolean = memScoped { if (value.isEmpty()) { return pw_encoder_write_string(raw, fieldNr, null, 0) } - return value.usePinned { - pw_encoder_write_string(raw, fieldNr, it.addressOf(0).reinterpret(), value.length) - } + val cStr = value.cstr + return pw_encoder_write_string(raw, fieldNr, cStr.ptr, cStr.size) } override fun writeBytes(fieldNr: Int, value: ByteArray): Boolean { @@ -122,9 +126,16 @@ internal class WireEncoderNative(private val sink: Sink): WireEncoder { } } - override fun flush() { - pw_encoder_flush(raw) + override fun writePackedFixed32(fieldNr: Int, value: UIntArray): Boolean { + if (value.isEmpty()) { + return pw_encoder_write_bytes(raw, fieldNr, null, 0) + } + val bytes = value.size * UInt.SIZE_BYTES + return value.usePinned { + pw_encoder_write_bytes(raw, fieldNr, it.addressOf(0), bytes) + } } + } internal actual fun WireEncoder(sink: Sink): WireEncoder = WireEncoderNative(sink) \ No newline at end of file diff --git a/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/WireCodecTest.kt b/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/WireCodecTest.kt index 26c61a8d7..d30c24f53 100644 --- a/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/WireCodecTest.kt +++ b/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/WireCodecTest.kt @@ -5,7 +5,12 @@ package kotlinx.rpc.grpc.internal import kotlinx.cinterop.ExperimentalForeignApi +import kotlinx.cinterop.addressOf +import kotlinx.cinterop.convert +import kotlinx.cinterop.readBytes +import kotlinx.cinterop.usePinned import kotlinx.io.Buffer +import platform.posix.memcpy import kotlin.experimental.ExperimentalNativeApi import kotlin.test.Test import kotlin.test.assertEquals @@ -609,6 +614,39 @@ class WireCodecTest { assertNotNull(tag) assertEquals(1, tag.fieldNr) assertEquals(WireType.LENGTH_DELIMITED, tag.wireType) + + val actualBytes = decoder.readBytes() + assertNotNull(actualBytes) + assertEquals(1000000, actualBytes.size) + assertTrue(bytes.contentEquals(actualBytes)) + + decoder.close() + assertTrue(buffer.exhausted()) + } + + @Test + fun testPackedFixed32EncodeDecode() { + val buffer = Buffer() + val encoder = WireEncoder(buffer) + + val ints = UIntArray(100000) { it.toUInt() } + + assertTrue(encoder.writePackedFixed32(1, ints)) + encoder.flush() + + val decoder = WireDecoder(buffer) + val tag = decoder.readTag() + assertNotNull(tag) + assertEquals(1, tag.fieldNr) + assertEquals(WireType.LENGTH_DELIMITED, tag.wireType) + + val actualInts = decoder.readPackedFixed32() + assertNotNull(actualInts) + assertEquals(ints.size, actualInts.size) + assertEquals(ints.toList(), actualInts) + + decoder.close() + assertTrue(buffer.exhausted()) } @Test diff --git a/versions-root/libs.versions.toml b/versions-root/libs.versions.toml index 49b12aac5..3be9fa2ec 100644 --- a/versions-root/libs.versions.toml +++ b/versions-root/libs.versions.toml @@ -21,6 +21,7 @@ intellij = "241.19416.19" gradle-doctor = "0.11.0" kotlinx-browser = "0.3" kotlinx-io = "0.8.0" +kotlinx-collections = "0.4.0" dokka = "2.0.0" puppeteer = "24.9.0" atomicfu = "0.29.0" @@ -61,6 +62,7 @@ serialization-plugin = { module = "org.jetbrains.kotlin:kotlin-serialization-com serialization-plugin-forIde = { module = "org.jetbrains.kotlin:kotlinx-serialization-compiler-plugin-for-ide", version.ref = "kotlin-compiler" } kotlinx-browser = { module = "org.jetbrains.kotlinx:kotlinx-browser", version.ref = "kotlinx-browser" } kotlinx-io-core = { module = "org.jetbrains.kotlinx:kotlinx-io-core", version.ref = "kotlinx-io" } +kotlinx-collections-immutable = { module = "org.jetbrains.kotlinx:kotlinx-collections-immutable", version.ref = "kotlinx-collections"} # serialization serialization-core = { module = "org.jetbrains.kotlinx:kotlinx-serialization-core", version.ref = "serialization" } From d62bce3ca7b541698242d623d23a95265dac0d14 Mon Sep 17 00:00:00 2001 From: Johannes Zottele Date: Fri, 25 Jul 2025 12:36:05 +0200 Subject: [PATCH 11/16] grpc-native: Implement all packed fixed sized field decoding Signed-off-by: Johannes Zottele --- .../kotlinx/rpc/grpc/internal/WireDecoder.kt | 7 +- .../kotlinx/rpc/grpc/internal/WireEncoder.kt | 10 +- .../rpc/grpc/internal/WireDecoder.native.kt | 104 +++++++++++++---- .../rpc/grpc/internal/WireEncoder.native.kt | 55 ++++++--- .../rpc/grpc/internal/WireCodecTest.kt | 108 ++++++++++++------ 5 files changed, 208 insertions(+), 76 deletions(-) diff --git a/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.kt b/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.kt index cfbf70a7e..1b0c38da8 100644 --- a/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.kt +++ b/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.kt @@ -28,7 +28,7 @@ import kotlinx.io.Buffer * } * ``` */ -internal interface WireDecoder: AutoCloseable { +internal interface WireDecoder : AutoCloseable { fun readTag(): KTag? fun readBool(): Boolean? fun readInt32(): Int? @@ -47,6 +47,11 @@ internal interface WireDecoder: AutoCloseable { fun readString(): String? fun readBytes(): ByteArray? fun readPackedFixed32(): List? + fun readPackedFixed64(): List? + fun readPackedSFixed32(): List? + fun readPackedSFixed64(): List? + fun readPackedFloat(): List? + fun readPackedDouble(): List? } /** diff --git a/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.kt b/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.kt index ba82bc399..d8d5fccbc 100644 --- a/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.kt +++ b/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.kt @@ -14,7 +14,9 @@ import kotlinx.io.Sink * * [flush] must be called to ensure that all data is written to the [Sink]. */ +@OptIn(ExperimentalUnsignedTypes::class) internal interface WireEncoder { + fun flush() fun writeBool(field: Int, value: Boolean): Boolean fun writeInt32(fieldNr: Int, value: Int): Boolean fun writeInt64(fieldNr: Int, value: Long): Boolean @@ -29,10 +31,14 @@ internal interface WireEncoder { fun writeFloat(fieldNr: Int, value: Float): Boolean fun writeDouble(fieldNr: Int, value: Double): Boolean fun writeEnum(fieldNr: Int, value: Int): Boolean - fun writeString(fieldNr: Int, value: String): Boolean - fun flush() fun writeBytes(fieldNr: Int, value: ByteArray): Boolean + fun writeString(fieldNr: Int, value: String): Boolean fun writePackedFixed32(fieldNr: Int, value: UIntArray): Boolean + fun writePackedFixed64(fieldNr: Int, value: ULongArray): Boolean + fun writePackedSFixed32(fieldNr: Int, value: IntArray): Boolean + fun writePackedSFixed64(fieldNr: Int, value: LongArray): Boolean + fun writePackedFloat(fieldNr: Int, value: FloatArray): Boolean + fun writePackedDouble(fieldNr: Int, value: DoubleArray): Boolean } diff --git a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.native.kt b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.native.kt index 5ed15b2a7..1c40e7686 100644 --- a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.native.kt +++ b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.native.kt @@ -11,10 +11,11 @@ import libprotowire.* import kotlin.experimental.ExperimentalNativeApi import kotlin.math.min +// maximum buffer size to allocate as contiguous memory in bytes private const val MAX_PACKED_BULK_SIZE: Int = 1_000_000 @OptIn(ExperimentalForeignApi::class, ExperimentalNativeApi::class) -internal class WireDecoderNative(private val source: Buffer): WireDecoder { +internal class WireDecoderNative(private val source: Buffer) : WireDecoder { // wraps the source in a class that allows to pass data from the source buffer to the C++ encoder // without copying it to an intermediate byte array. @@ -201,6 +202,48 @@ internal class WireDecoderNative(private val source: Buffer): WireDecoder { return bytes } + override fun readPackedFixed32() = readPackedFixedInternal( + UInt.SIZE_BYTES, + ::UIntArray, + Pinned::addressOf, + UIntArray::asList, + ) + + override fun readPackedFixed64() = readPackedFixedInternal( + ULong.SIZE_BYTES, + ::ULongArray, + Pinned::addressOf, + ULongArray::asList, + ) + + override fun readPackedSFixed32() = readPackedFixedInternal( + Int.SIZE_BYTES, + ::IntArray, + Pinned::addressOf, + IntArray::asList, + ) + + override fun readPackedSFixed64() = readPackedFixedInternal( + Long.SIZE_BYTES, + ::LongArray, + Pinned::addressOf, + LongArray::asList, + ) + + override fun readPackedFloat() = readPackedFixedInternal( + Float.SIZE_BYTES, + ::FloatArray, + Pinned::addressOf, + FloatArray::asList, + ) + + override fun readPackedDouble() = readPackedFixedInternal( + Double.SIZE_BYTES, + ::DoubleArray, + Pinned::addressOf, + DoubleArray::asList, + ) + /* * Based on the length of the packed repeated field, one of two list strategies is chosen. * If the length is less or equal a specific threshold (MAX_PACKED_BULK_SIZE), @@ -210,39 +253,52 @@ internal class WireDecoderNative(private val source: Buffer): WireDecoder { * * Note that this implementation assumes a little endian memory order. */ - override fun readPackedFixed32(): List? { + private inline fun readPackedFixedInternal( + sizeBytes: Int, + crossinline createArray: (Int) -> R, + crossinline getAddress: Pinned.(Int) -> COpaquePointer, + crossinline asList: (R) -> List + ): List? { + // fetch the size of the packed repeated field var byteLen = readInt32() ?: return null if (byteLen < 0) return null if (source.size < byteLen) return null - if (byteLen % UInt.SIZE_BYTES != 0 ) return null - val count = byteLen / UInt.SIZE_BYTES + if (byteLen % sizeBytes != 0) return null if (byteLen == 0) return emptyList() - if (count <= MAX_PACKED_BULK_SIZE) { - // this implementation assumes that the program is running on little endian machines. - val arr = UIntArray(count) - arr.usePinned { - pw_decoder_read_raw_bytes(raw, it.addressOf(0), byteLen) - } - return ArrayList(arr) - } else { - val bufByteLen = MAX_PACKED_BULK_SIZE - val bufLen = bufByteLen / UInt.SIZE_BYTES - val buffer = UIntArray(bufLen) - var list = persistentListOf() - buffer.usePinned { + // allocate the buffer array (has at most MAX_PACKED_BULK_SIZE bytes) + val bufByteLen = minOf(byteLen, MAX_PACKED_BULK_SIZE) + val bufElemCount = bufByteLen / sizeBytes + val buffer = createArray(bufElemCount) + + buffer.usePinned { + val bufAddr = it.getAddress(0) + + if (byteLen == bufByteLen) { + // the whole packed field fits into the buffer -> copy into buffer and returns it as a list. + pw_decoder_read_raw_bytes(raw, bufAddr, byteLen) + return asList(buffer) + } else { + // the packed field is too large for the buffer, so we load it into a persistent list + var chunkedList = persistentListOf() + while (byteLen > 0) { - val written = min(bufByteLen, byteLen) - pw_decoder_read_raw_bytes(raw, it.addressOf(0), written) - list = if (written == bufByteLen) { - list.addAll(buffer) + // copy data into the buffer. + val copySize = min(bufByteLen, byteLen) + pw_decoder_read_raw_bytes(raw, bufAddr, copySize) + + // add buffer to the chunked list + chunkedList = if (copySize == bufByteLen) { + chunkedList.addAll(asList(buffer)) } else { - list.addAll(buffer.copyOfRange(0, written / UInt.SIZE_BYTES)) + chunkedList.addAll(asList(buffer).subList(0, copySize / sizeBytes)) } - byteLen -= written + + byteLen -= copySize } + + return chunkedList } - return list } } } diff --git a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.native.kt b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.native.kt index 3e7540cab..3dd4e0eb2 100644 --- a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.native.kt +++ b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.native.kt @@ -7,16 +7,14 @@ package kotlinx.rpc.grpc.internal import kotlinx.cinterop.* import kotlinx.io.Sink import libprotowire.* -import kotlin.collections.isEmpty import kotlin.experimental.ExperimentalNativeApi import kotlin.native.ref.createCleaner -import kotlin.text.isEmpty // TODO: Evaluate if we should implement a ZeroCopyOutputSink (similar to the ZeroCopyInputSource) // to reduce the number of copies during encoding. @OptIn(ExperimentalForeignApi::class, ExperimentalNativeApi::class) -internal class WireEncoderNative(private val sink: Sink): WireEncoder { +internal class WireEncoderNative(private val sink: Sink) : WireEncoder { /** * The context object provides a stable reference to the kotlin context. * This is required, as functions must be static and cannot capture environment references. @@ -32,6 +30,7 @@ internal class WireEncoderNative(private val sink: Sink): WireEncoder { // create context as a stable reference that can be passed to static function callback private val context = StableRef.create(this.Ctx()) + // construct encoder with a callback that calls write() on this.context internal val raw = run { pw_encoder_new(context.asCPointer(), staticCFunction { ctx, buf, size -> @@ -126,16 +125,46 @@ internal class WireEncoderNative(private val sink: Sink): WireEncoder { } } - override fun writePackedFixed32(fieldNr: Int, value: UIntArray): Boolean { - if (value.isEmpty()) { - return pw_encoder_write_bytes(raw, fieldNr, null, 0) - } - val bytes = value.size * UInt.SIZE_BYTES - return value.usePinned { - pw_encoder_write_bytes(raw, fieldNr, it.addressOf(0), bytes) - } - } + override fun writePackedFixed32(fieldNr: Int, value: UIntArray) = + writePackedInternal(fieldNr, value, UIntArray::size, UInt.SIZE_BYTES) + { it.addressOf(0) } + override fun writePackedFixed64(fieldNr: Int, value: ULongArray) = + writePackedInternal(fieldNr, value, ULongArray::size, ULong.SIZE_BYTES) + { it.addressOf(0) } + + override fun writePackedSFixed32(fieldNr: Int, value: IntArray) = + writePackedInternal(fieldNr, value, IntArray::size, Int.SIZE_BYTES) + { it.addressOf(0) } + + override fun writePackedSFixed64(fieldNr: Int, value: LongArray) = + writePackedInternal(fieldNr, value, LongArray::size, Long.SIZE_BYTES) + { it.addressOf(0) } + + override fun writePackedFloat(fieldNr: Int, value: FloatArray) = + writePackedInternal(fieldNr, value, FloatArray::size, Float.SIZE_BYTES) + { it.addressOf(0) } + + override fun writePackedDouble(fieldNr: Int, value: DoubleArray) = + writePackedInternal(fieldNr, value, DoubleArray::size, Double.SIZE_BYTES) + { it.addressOf(0) } } -internal actual fun WireEncoder(sink: Sink): WireEncoder = WireEncoderNative(sink) \ No newline at end of file +internal actual fun WireEncoder(sink: Sink): WireEncoder = WireEncoderNative(sink) + + +@OptIn(ExperimentalForeignApi::class) +private inline fun WireEncoderNative.writePackedInternal( + fieldNr: Int, + value: A, + crossinline sizeOf: A.() -> Int, + sizeBytes: Int, + crossinline ptr: (Pinned) -> COpaquePointer +): Boolean { + val len = sizeOf(value) + if (len == 0) return pw_encoder_write_bytes(raw, fieldNr, null, 0) + val bytes = len * sizeBytes + return value.usePinned { pinned -> + pw_encoder_write_bytes(raw, fieldNr, ptr(pinned), bytes) + } +} \ No newline at end of file diff --git a/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/WireCodecTest.kt b/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/WireCodecTest.kt index d30c24f53..fe9009cbb 100644 --- a/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/WireCodecTest.kt +++ b/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/WireCodecTest.kt @@ -5,19 +5,9 @@ package kotlinx.rpc.grpc.internal import kotlinx.cinterop.ExperimentalForeignApi -import kotlinx.cinterop.addressOf -import kotlinx.cinterop.convert -import kotlinx.cinterop.readBytes -import kotlinx.cinterop.usePinned import kotlinx.io.Buffer -import platform.posix.memcpy import kotlin.experimental.ExperimentalNativeApi -import kotlin.test.Test -import kotlin.test.assertEquals -import kotlin.test.assertFalse -import kotlin.test.assertNotNull -import kotlin.test.assertNull -import kotlin.test.assertTrue +import kotlin.test.* // TODO: Move this to the commonTest @OptIn(ExperimentalForeignApi::class, ExperimentalNativeApi::class) @@ -624,31 +614,6 @@ class WireCodecTest { assertTrue(buffer.exhausted()) } - @Test - fun testPackedFixed32EncodeDecode() { - val buffer = Buffer() - val encoder = WireEncoder(buffer) - - val ints = UIntArray(100000) { it.toUInt() } - - assertTrue(encoder.writePackedFixed32(1, ints)) - encoder.flush() - - val decoder = WireDecoder(buffer) - val tag = decoder.readTag() - assertNotNull(tag) - assertEquals(1, tag.fieldNr) - assertEquals(WireType.LENGTH_DELIMITED, tag.wireType) - - val actualInts = decoder.readPackedFixed32() - assertNotNull(actualInts) - assertEquals(ints.size, actualInts.size) - assertEquals(ints.toList(), actualInts) - - decoder.close() - assertTrue(buffer.exhausted()) - } - @Test fun testDoubleEncodeDecode() { val fieldNr = 21 @@ -698,4 +663,75 @@ class WireCodecTest { decoder.close() assertTrue(buffer.exhausted()) } + + + private inline fun runPackedTest( + arr: A, + crossinline write: WireEncoder.(Int, A) -> Boolean, + crossinline read: WireDecoder.() -> List?, + crossinline asList: (A) -> List + ) { + val buf = Buffer() + with(WireEncoder(buf)) { + assertTrue(write(1, arr)) + flush() + } + WireDecoder(buf).use { dec -> + dec.readTag()!!.apply { + assertEquals(1, fieldNr) + assertEquals(WireType.LENGTH_DELIMITED, wireType) + } + val test = dec.read() + assertEquals(asList(arr), test) + } + assertTrue(buf.exhausted()) + } + + @Test + fun testPackedFixed32() = runPackedTest( + UIntArray(1_000_000) { UInt.MAX_VALUE + it.toUInt() }, + WireEncoder::writePackedFixed32, + WireDecoder::readPackedFixed32, + UIntArray::asList + ) + + @Test + fun testPackedFixed64() = runPackedTest( + ULongArray(1_000_000) { UInt.MAX_VALUE + it.toULong() }, + WireEncoder::writePackedFixed64, + WireDecoder::readPackedFixed64, + ULongArray::asList + ) + + @Test + fun testPackedSFixed32() = runPackedTest( + IntArray(1_000_000) { Int.MAX_VALUE + it }, + WireEncoder::writePackedSFixed32, + WireDecoder::readPackedSFixed32, + IntArray::asList + ) + + @Test + fun testPackedSFixed64() = runPackedTest( + LongArray(1_000_000) { Long.MAX_VALUE + it }, + WireEncoder::writePackedSFixed64, + WireDecoder::readPackedSFixed64, + LongArray::asList + ) + + @Test + fun testPackedFloat() = runPackedTest( + FloatArray(1_000_000) { it.toFloat() / 3.3f * ((it and 1) * 2 - 1) }, + WireEncoder::writePackedFloat, + WireDecoder::readPackedFloat, + FloatArray::asList + ) + + @Test + fun testPackedDouble() = runPackedTest( + DoubleArray(1_000_000) { it.toDouble() / 3.3 * ((it and 1) * 2 - 1) }, + WireEncoder::writePackedDouble, + WireDecoder::readPackedDouble, + DoubleArray::asList + ) } From 939ed809b30de7db6ec2f1418c515a239c850b06 Mon Sep 17 00:00:00 2001 From: Johannes Zottele Date: Fri, 25 Jul 2025 15:24:38 +0200 Subject: [PATCH 12/16] grpc-native: Change packed input value type to List Signed-off-by: Johannes Zottele --- .../kotlin/kotlinx/rpc/grpc/internal/KTag.kt | 7 +++ .../kotlinx/rpc/grpc/internal/WireEncoder.kt | 12 ++--- .../rpc/grpc/internal/WireDecoder.native.kt | 14 +++-- .../rpc/grpc/internal/WireEncoder.native.kt | 54 +++++++++---------- .../rpc/grpc/internal/WireCodecTest.kt | 38 ++++++------- grpc/grpcpp-c/include/protowire.h | 17 ++++++ grpc/grpcpp-c/src/protowire.cpp | 34 ++++++++++++ 7 files changed, 114 insertions(+), 62 deletions(-) diff --git a/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/KTag.kt b/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/KTag.kt index e4c7e6288..474ec4c36 100644 --- a/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/KTag.kt +++ b/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/KTag.kt @@ -4,6 +4,8 @@ package kotlinx.rpc.grpc.internal +import kotlinx.rpc.grpc.internal.KTag.Companion.K_TAG_TYPE_BITS + internal enum class WireType { VARINT, // 0 FIXED64, // 1 @@ -22,11 +24,16 @@ internal data class KTag(val fieldNr: Int, val wireType: WireType) { companion object { // Number of bits in a tag which identify the wire type. const val K_TAG_TYPE_BITS: Int = 3; + // Mask for those bits. (just 0b111) val K_TAG_TYPE_MASK: UInt = (1u shl K_TAG_TYPE_BITS) - 1u } } +internal fun KTag.toRawKTag(): UInt { + return (fieldNr.toUInt() shl K_TAG_TYPE_BITS) or wireType.ordinal.toUInt() +} + internal fun KTag.Companion.from(rawKTag: UInt): KTag? { val type = (rawKTag and K_TAG_TYPE_MASK).toInt() val field = (rawKTag shr K_TAG_TYPE_BITS).toInt() diff --git a/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.kt b/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.kt index d8d5fccbc..9d1ada493 100644 --- a/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.kt +++ b/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.kt @@ -33,12 +33,12 @@ internal interface WireEncoder { fun writeEnum(fieldNr: Int, value: Int): Boolean fun writeBytes(fieldNr: Int, value: ByteArray): Boolean fun writeString(fieldNr: Int, value: String): Boolean - fun writePackedFixed32(fieldNr: Int, value: UIntArray): Boolean - fun writePackedFixed64(fieldNr: Int, value: ULongArray): Boolean - fun writePackedSFixed32(fieldNr: Int, value: IntArray): Boolean - fun writePackedSFixed64(fieldNr: Int, value: LongArray): Boolean - fun writePackedFloat(fieldNr: Int, value: FloatArray): Boolean - fun writePackedDouble(fieldNr: Int, value: DoubleArray): Boolean + fun writePackedFixed32(fieldNr: Int, value: List): Boolean + fun writePackedFixed64(fieldNr: Int, value: List): Boolean + fun writePackedSFixed32(fieldNr: Int, value: List): Boolean + fun writePackedSFixed64(fieldNr: Int, value: List): Boolean + fun writePackedFloat(fieldNr: Int, value: List): Boolean + fun writePackedDouble(fieldNr: Int, value: List): Boolean } diff --git a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.native.kt b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.native.kt index 1c40e7686..152d23279 100644 --- a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.native.kt +++ b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.native.kt @@ -10,6 +10,7 @@ import kotlinx.io.Buffer import libprotowire.* import kotlin.experimental.ExperimentalNativeApi import kotlin.math.min +import kotlin.native.ref.createCleaner // maximum buffer size to allocate as contiguous memory in bytes private const val MAX_PACKED_BULK_SIZE: Int = 1_000_000 @@ -44,14 +45,17 @@ internal class WireDecoderNative(private val source: Buffer) : WireDecoder { pw_decoder_new(zeroCopyCInput) ?: error("Failed to create proto wire decoder") } + + val rawCleaner = createCleaner(raw) { + pw_decoder_delete(it) + } override fun close() { - // delete the underlying decoder. - // this will also fix the position in the source buffer + // this will fix the position in the source buffer // (done by deconstructor of CodedInputStream) - pw_decoder_delete(raw) - // close zero inputs on close + pw_decoder_close(raw) + zeroCopyInput.get().close() zeroCopyInput.dispose() } @@ -243,7 +247,7 @@ internal class WireDecoderNative(private val source: Buffer) : WireDecoder { Pinned::addressOf, DoubleArray::asList, ) - + /* * Based on the length of the packed repeated field, one of two list strategies is chosen. * If the length is less or equal a specific threshold (MAX_PACKED_BULK_SIZE), diff --git a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.native.kt b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.native.kt index 3dd4e0eb2..f4cda86d3 100644 --- a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.native.kt +++ b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.native.kt @@ -125,46 +125,44 @@ internal class WireEncoderNative(private val sink: Sink) : WireEncoder { } } - override fun writePackedFixed32(fieldNr: Int, value: UIntArray) = - writePackedInternal(fieldNr, value, UIntArray::size, UInt.SIZE_BYTES) - { it.addressOf(0) } + override fun writePackedFixed32(fieldNr: Int, value: List) = + writePackedInternal(fieldNr, value, UInt.SIZE_BYTES, ::pw_encoder_write_fixed32_no_tag) - override fun writePackedFixed64(fieldNr: Int, value: ULongArray) = - writePackedInternal(fieldNr, value, ULongArray::size, ULong.SIZE_BYTES) - { it.addressOf(0) } + override fun writePackedFixed64(fieldNr: Int, value: List) = + writePackedInternal(fieldNr, value, ULong.SIZE_BYTES, ::pw_encoder_write_fixed64_no_tag) - override fun writePackedSFixed32(fieldNr: Int, value: IntArray) = - writePackedInternal(fieldNr, value, IntArray::size, Int.SIZE_BYTES) - { it.addressOf(0) } + override fun writePackedSFixed32(fieldNr: Int, value: List) = + writePackedInternal(fieldNr, value, Int.SIZE_BYTES, ::pw_encoder_write_sfixed32_no_tag) - override fun writePackedSFixed64(fieldNr: Int, value: LongArray) = - writePackedInternal(fieldNr, value, LongArray::size, Long.SIZE_BYTES) - { it.addressOf(0) } + override fun writePackedSFixed64(fieldNr: Int, value: List) = + writePackedInternal(fieldNr, value, Long.SIZE_BYTES, ::pw_encoder_write_sfixed64_no_tag) - override fun writePackedFloat(fieldNr: Int, value: FloatArray) = - writePackedInternal(fieldNr, value, FloatArray::size, Float.SIZE_BYTES) - { it.addressOf(0) } + override fun writePackedFloat(fieldNr: Int, value: List) = + writePackedInternal(fieldNr, value, Float.SIZE_BYTES, ::pw_encoder_write_float_no_tag) - override fun writePackedDouble(fieldNr: Int, value: DoubleArray) = - writePackedInternal(fieldNr, value, DoubleArray::size, Double.SIZE_BYTES) - { it.addressOf(0) } + override fun writePackedDouble(fieldNr: Int, value: List) = + writePackedInternal(fieldNr, value, Double.SIZE_BYTES, ::pw_encoder_write_double_no_tag) } internal actual fun WireEncoder(sink: Sink): WireEncoder = WireEncoderNative(sink) @OptIn(ExperimentalForeignApi::class) -private inline fun WireEncoderNative.writePackedInternal( +private inline fun WireEncoderNative.writePackedInternal( fieldNr: Int, - value: A, - crossinline sizeOf: A.() -> Int, - sizeBytes: Int, - crossinline ptr: (Pinned) -> COpaquePointer + value: List, + byteSize: Int, + crossinline writer: (CValuesRef?, T) -> Boolean ): Boolean { - val len = sizeOf(value) - if (len == 0) return pw_encoder_write_bytes(raw, fieldNr, null, 0) - val bytes = len * sizeBytes - return value.usePinned { pinned -> - pw_encoder_write_bytes(raw, fieldNr, ptr(pinned), bytes) + val ktag = KTag(fieldNr, WireType.LENGTH_DELIMITED).toRawKTag() + pw_encoder_write_tag(raw, ktag) + // write the field size of the packed field + val fieldSize = value.size * byteSize; + pw_encoder_write_int32_no_tag(raw, fieldSize) + for (v in value) { + if (!writer(raw, v)) { + return false + } } + return true } \ No newline at end of file diff --git a/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/WireCodecTest.kt b/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/WireCodecTest.kt index fe9009cbb..01765c419 100644 --- a/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/WireCodecTest.kt +++ b/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/WireCodecTest.kt @@ -664,16 +664,14 @@ class WireCodecTest { assertTrue(buffer.exhausted()) } - - private inline fun runPackedTest( - arr: A, - crossinline write: WireEncoder.(Int, A) -> Boolean, - crossinline read: WireDecoder.() -> List?, - crossinline asList: (A) -> List + private fun runPackedTest( + list: List, + write: WireEncoder.(Int, List) -> Boolean, + read: WireDecoder.() -> List?, ) { val buf = Buffer() with(WireEncoder(buf)) { - assertTrue(write(1, arr)) + assertTrue(write(1, list)) flush() } WireDecoder(buf).use { dec -> @@ -682,56 +680,50 @@ class WireCodecTest { assertEquals(WireType.LENGTH_DELIMITED, wireType) } val test = dec.read() - assertEquals(asList(arr), test) + assertEquals(list, test) } assertTrue(buf.exhausted()) } @Test fun testPackedFixed32() = runPackedTest( - UIntArray(1_000_000) { UInt.MAX_VALUE + it.toUInt() }, + List(1_000_000) { UInt.MAX_VALUE + it.toUInt() }, WireEncoder::writePackedFixed32, - WireDecoder::readPackedFixed32, - UIntArray::asList + WireDecoder::readPackedFixed32 ) @Test fun testPackedFixed64() = runPackedTest( - ULongArray(1_000_000) { UInt.MAX_VALUE + it.toULong() }, + List(1_000_000) { UInt.MAX_VALUE + it.toULong() }, WireEncoder::writePackedFixed64, WireDecoder::readPackedFixed64, - ULongArray::asList ) @Test fun testPackedSFixed32() = runPackedTest( - IntArray(1_000_000) { Int.MAX_VALUE + it }, + List(1_000_000) { Int.MAX_VALUE + it }, WireEncoder::writePackedSFixed32, - WireDecoder::readPackedSFixed32, - IntArray::asList + WireDecoder::readPackedSFixed32 ) @Test fun testPackedSFixed64() = runPackedTest( - LongArray(1_000_000) { Long.MAX_VALUE + it }, + List(1_000_000) { Long.MAX_VALUE + it }, WireEncoder::writePackedSFixed64, - WireDecoder::readPackedSFixed64, - LongArray::asList + WireDecoder::readPackedSFixed64 ) @Test fun testPackedFloat() = runPackedTest( - FloatArray(1_000_000) { it.toFloat() / 3.3f * ((it and 1) * 2 - 1) }, + List(1_000_000) { it.toFloat() / 3.3f * ((it and 1) * 2 - 1) }, WireEncoder::writePackedFloat, WireDecoder::readPackedFloat, - FloatArray::asList ) @Test fun testPackedDouble() = runPackedTest( - DoubleArray(1_000_000) { it.toDouble() / 3.3 * ((it and 1) * 2 - 1) }, + List(1_000_000) { it.toDouble() / 3.3 * ((it and 1) * 2 - 1) }, WireEncoder::writePackedDouble, WireDecoder::readPackedDouble, - DoubleArray::asList ) } diff --git a/grpc/grpcpp-c/include/protowire.h b/grpc/grpcpp-c/include/protowire.h index ddd97fa84..bcd12de64 100644 --- a/grpc/grpcpp-c/include/protowire.h +++ b/grpc/grpcpp-c/include/protowire.h @@ -51,6 +51,22 @@ extern "C" { bool pw_encoder_write_string(pw_encoder_t *self, int field_no, const char *data, int size); bool pw_encoder_write_bytes(pw_encoder_t *self, int field_no, const void *data, int size); + // No tag writers + bool pw_encoder_write_tag(pw_encoder_t *self, uint32_t tag); + bool pw_encoder_write_bool_no_tag(pw_encoder_t *self, bool value); + bool pw_encoder_write_int32_no_tag(pw_encoder_t *self, int32_t value); + bool pw_encoder_write_int64_no_tag(pw_encoder_t *self, int64_t value); + bool pw_encoder_write_uint32_no_tag(pw_encoder_t *self, uint32_t value); + bool pw_encoder_write_uint64_no_tag(pw_encoder_t *self, uint64_t value); + bool pw_encoder_write_sint32_no_tag(pw_encoder_t *self, int32_t value); + bool pw_encoder_write_sint64_no_tag(pw_encoder_t *self, int64_t value); + bool pw_encoder_write_fixed32_no_tag(pw_encoder_t *self, uint32_t value); + bool pw_encoder_write_fixed64_no_tag(pw_encoder_t *self, uint64_t value); + bool pw_encoder_write_sfixed32_no_tag(pw_encoder_t *self, int32_t value); + bool pw_encoder_write_sfixed64_no_tag(pw_encoder_t *self, int64_t value); + bool pw_encoder_write_float_no_tag(pw_encoder_t *self, float value); + bool pw_encoder_write_double_no_tag(pw_encoder_t *self, double value); + //// WIRE DECODER //// @@ -78,6 +94,7 @@ extern "C" { */ pw_decoder_t * pw_decoder_new(pw_zero_copy_input_t zero_copy_input); void pw_decoder_delete(pw_decoder_t *self); + void pw_decoder_close(pw_decoder_t *self); uint32_t pw_decoder_read_tag(pw_decoder_t *self); bool pw_decoder_read_bool(pw_decoder_t *self, bool *value); diff --git a/grpc/grpcpp-c/src/protowire.cpp b/grpc/grpcpp-c/src/protowire.cpp index eebaa2e02..6f918842f 100644 --- a/grpc/grpcpp-c/src/protowire.cpp +++ b/grpc/grpcpp-c/src/protowire.cpp @@ -180,6 +180,35 @@ extern "C" { return check(self); } + bool pw_encoder_write_tag(pw_encoder_t *self, uint32_t tag) { + self->cos.WriteTag(tag); + return check(self); + } + +#define WRITE_FIELD_NO_TAG( funcSuffix, wireTy, cTy) \ +bool pw_encoder_write_##funcSuffix##_no_tag(pw_encoder_t *self, cTy value) { \ +WireFormatLite::Write##wireTy##NoTag(value, &self->cos); \ +return check(self); \ +} + + WRITE_FIELD_NO_TAG( bool, Bool, bool) + WRITE_FIELD_NO_TAG( int32, Int32, int32_t) + WRITE_FIELD_NO_TAG( int64, Int64, int64_t) + WRITE_FIELD_NO_TAG( uint32, UInt32, uint32_t) + WRITE_FIELD_NO_TAG( uint64, UInt64, uint64_t) + WRITE_FIELD_NO_TAG( sint32, SInt32, int32_t) + WRITE_FIELD_NO_TAG( sint64, SInt64, int64_t) + WRITE_FIELD_NO_TAG( fixed32, Fixed32, uint32_t) + WRITE_FIELD_NO_TAG( fixed64, Fixed64, uint64_t) + WRITE_FIELD_NO_TAG( sfixed32, SFixed32, int32_t) + WRITE_FIELD_NO_TAG( sfixed64, SFixed64, int64_t) + WRITE_FIELD_NO_TAG( float, Float, float) + WRITE_FIELD_NO_TAG( double, Double, double) + WRITE_FIELD_NO_TAG( enum, Enum, int) + + + /// DECODER IMPLEMENATION /// + pw_decoder_t *pw_decoder_new(pw_zero_copy_input_t zero_copy_input) { return new pw_decoder_t(zero_copy_input); } @@ -188,6 +217,11 @@ extern "C" { delete self; } + void pw_decoder_close(pw_decoder_t *self) { + // the deconstructor backs the stream up to the current position. + self->cis.~CodedInputStream(); + } + uint32_t pw_decoder_read_tag(pw_decoder_t *self) { return self->cis.ReadTag(); } From f5f8f09e6d8d3ef5f2e4cf9943dda2bfaae047ac Mon Sep 17 00:00:00 2001 From: Johannes Zottele Date: Fri, 25 Jul 2025 18:32:09 +0200 Subject: [PATCH 13/16] grpc-native: Add variable length packed field encoding Signed-off-by: Johannes Zottele --- .../kotlinx/rpc/grpc/internal/WireDecoder.kt | 8 ++ .../kotlinx/rpc/grpc/internal/WireEncoder.kt | 9 ++ .../kotlinx/rpc/grpc/internal/WireSize.kt | 24 +++++ .../kotlinx/rpc/grpc/internal/WireSize.js.kt | 29 ++++++ .../kotlinx/rpc/grpc/internal/WireSize.jvm.kt | 29 ++++++ .../rpc/grpc/internal/WireDecoder.native.kt | 32 ++++++- .../rpc/grpc/internal/WireEncoder.native.kt | 40 ++++++-- .../rpc/grpc/internal/WireSize.native.kt | 18 ++++ .../rpc/grpc/internal/WireCodecTest.kt | 93 +++++++++++++++++-- .../rpc/grpc/internal/WireSize.wasmJs.kt | 29 ++++++ grpc/grpcpp-c/include/protowire.h | 15 ++- grpc/grpcpp-c/src/protowire.cpp | 32 +++++++ 12 files changed, 339 insertions(+), 19 deletions(-) create mode 100644 grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireSize.kt create mode 100644 grpc/grpc-core/src/jsMain/kotlin/kotlinx/rpc/grpc/internal/WireSize.js.kt create mode 100644 grpc/grpc-core/src/jvmMain/kotlin/kotlinx/rpc/grpc/internal/WireSize.jvm.kt create mode 100644 grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireSize.native.kt create mode 100644 grpc/grpc-core/src/wasmJsMain/kotlin/kotlinx/rpc/grpc/internal/WireSize.wasmJs.kt diff --git a/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.kt b/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.kt index 1b0c38da8..84c2fd0c6 100644 --- a/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.kt +++ b/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.kt @@ -46,12 +46,20 @@ internal interface WireDecoder : AutoCloseable { fun readEnum(): Int? fun readString(): String? fun readBytes(): ByteArray? + fun readPackedBool(): List? + fun readPackedInt32(): List? + fun readPackedInt64(): List? + fun readPackedSInt32(): List? + fun readPackedSInt64(): List? + fun readPackedUInt32(): List? + fun readPackedUInt64(): List? fun readPackedFixed32(): List? fun readPackedFixed64(): List? fun readPackedSFixed32(): List? fun readPackedSFixed64(): List? fun readPackedFloat(): List? fun readPackedDouble(): List? + fun readPackedEnum(): List? } /** diff --git a/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.kt b/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.kt index 9d1ada493..918b30bef 100644 --- a/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.kt +++ b/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.kt @@ -33,12 +33,21 @@ internal interface WireEncoder { fun writeEnum(fieldNr: Int, value: Int): Boolean fun writeBytes(fieldNr: Int, value: ByteArray): Boolean fun writeString(fieldNr: Int, value: String): Boolean + fun writePackedBool(fieldNr: Int, value: List, fieldSize: Int): Boolean + fun writePackedInt32(fieldNr: Int, value: List, fieldSize: Int): Boolean + fun writePackedInt64(fieldNr: Int, value: List, fieldSize: Int): Boolean + fun writePackedUInt32(fieldNr: Int, value: List, fieldSize: Int): Boolean + fun writePackedUInt64(fieldNr: Int, value: List, fieldSize: Int): Boolean + fun writePackedSInt32(fieldNr: Int, value: List, fieldSize: Int): Boolean + fun writePackedSInt64(fieldNr: Int, value: List, fieldSize: Int): Boolean fun writePackedFixed32(fieldNr: Int, value: List): Boolean fun writePackedFixed64(fieldNr: Int, value: List): Boolean fun writePackedSFixed32(fieldNr: Int, value: List): Boolean fun writePackedSFixed64(fieldNr: Int, value: List): Boolean fun writePackedFloat(fieldNr: Int, value: List): Boolean fun writePackedDouble(fieldNr: Int, value: List): Boolean + fun writePackedEnum(fieldNr: Int, value: List, fieldSize: Int) = + writePackedInt32(fieldNr, value, fieldSize) } diff --git a/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireSize.kt b/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireSize.kt new file mode 100644 index 000000000..161a4fe92 --- /dev/null +++ b/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireSize.kt @@ -0,0 +1,24 @@ +/* + * Copyright 2023-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.rpc.grpc.internal + +internal object WireSize + +internal expect fun WireSize.int32(value: Int): UInt +internal expect fun WireSize.int64(value: Long): UInt +internal expect fun WireSize.uInt32(value: UInt): UInt +internal expect fun WireSize.uInt64(value: ULong): UInt +internal expect fun WireSize.sInt32(value: Int): UInt +internal expect fun WireSize.sInt64(value: Long): UInt + +internal fun WireSize.bool(value: Boolean) = int32(if (value) 1 else 0) +internal fun WireSize.enum(value: Int) = int32(value) +internal fun WireSize.packedInt32(value: List) = value.sumOf { int32(it) } +internal fun WireSize.packedInt64(value: List) = value.sumOf { int64(it) } +internal fun WireSize.packedUInt32(value: List) = value.sumOf { uInt32(it) } +internal fun WireSize.packedUInt64(value: List) = value.sumOf { uInt64(it) } +internal fun WireSize.packedSInt32(value: List) = value.sumOf { sInt32(it) } +internal fun WireSize.packedSInt64(value: List) = value.sumOf { sInt64(it) } +internal fun WireSize.packedEnum(value: List) = value.sumOf { enum(it) } diff --git a/grpc/grpc-core/src/jsMain/kotlin/kotlinx/rpc/grpc/internal/WireSize.js.kt b/grpc/grpc-core/src/jsMain/kotlin/kotlinx/rpc/grpc/internal/WireSize.js.kt new file mode 100644 index 000000000..e70b9bd00 --- /dev/null +++ b/grpc/grpc-core/src/jsMain/kotlin/kotlinx/rpc/grpc/internal/WireSize.js.kt @@ -0,0 +1,29 @@ +/* + * Copyright 2023-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.rpc.grpc.internal + +internal actual fun WireSize.int32(value: Int): UInt { + TODO("Not yet implemented") +} + +internal actual fun WireSize.int64(value: Long): UInt { + TODO("Not yet implemented") +} + +internal actual fun WireSize.uInt32(value: UInt): UInt { + TODO("Not yet implemented") +} + +internal actual fun WireSize.uInt64(value: ULong): UInt { + TODO("Not yet implemented") +} + +internal actual fun WireSize.sInt32(value: Int): UInt { + TODO("Not yet implemented") +} + +internal actual fun WireSize.sInt64(value: Long): UInt { + TODO("Not yet implemented") +} \ No newline at end of file diff --git a/grpc/grpc-core/src/jvmMain/kotlin/kotlinx/rpc/grpc/internal/WireSize.jvm.kt b/grpc/grpc-core/src/jvmMain/kotlin/kotlinx/rpc/grpc/internal/WireSize.jvm.kt new file mode 100644 index 000000000..e70b9bd00 --- /dev/null +++ b/grpc/grpc-core/src/jvmMain/kotlin/kotlinx/rpc/grpc/internal/WireSize.jvm.kt @@ -0,0 +1,29 @@ +/* + * Copyright 2023-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.rpc.grpc.internal + +internal actual fun WireSize.int32(value: Int): UInt { + TODO("Not yet implemented") +} + +internal actual fun WireSize.int64(value: Long): UInt { + TODO("Not yet implemented") +} + +internal actual fun WireSize.uInt32(value: UInt): UInt { + TODO("Not yet implemented") +} + +internal actual fun WireSize.uInt64(value: ULong): UInt { + TODO("Not yet implemented") +} + +internal actual fun WireSize.sInt32(value: Int): UInt { + TODO("Not yet implemented") +} + +internal actual fun WireSize.sInt64(value: Long): UInt { + TODO("Not yet implemented") +} \ No newline at end of file diff --git a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.native.kt b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.native.kt index 152d23279..af980e757 100644 --- a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.native.kt +++ b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.native.kt @@ -45,7 +45,7 @@ internal class WireDecoderNative(private val source: Buffer) : WireDecoder { pw_decoder_new(zeroCopyCInput) ?: error("Failed to create proto wire decoder") } - + val rawCleaner = createCleaner(raw) { pw_decoder_delete(it) } @@ -206,6 +206,15 @@ internal class WireDecoderNative(private val source: Buffer) : WireDecoder { return bytes } + override fun readPackedBool() = readPackedVarInternal(this::readBool) + override fun readPackedInt32() = readPackedVarInternal(this::readInt32) + override fun readPackedInt64() = readPackedVarInternal(this::readInt64) + override fun readPackedUInt32() = readPackedVarInternal(this::readUInt32) + override fun readPackedUInt64() = readPackedVarInternal(this::readUInt64) + override fun readPackedSInt32() = readPackedVarInternal(this::readSInt32) + override fun readPackedSInt64() = readPackedVarInternal(this::readSInt64) + override fun readPackedEnum() = readPackedVarInternal(this::readEnum) + override fun readPackedFixed32() = readPackedFixedInternal( UInt.SIZE_BYTES, ::UIntArray, @@ -248,6 +257,27 @@ internal class WireDecoderNative(private val source: Buffer) : WireDecoder { DoubleArray::asList, ) + private inline fun readPackedVarInternal( + crossinline readFn: () -> T? + ): List? { + val byteLen = readInt32() ?: return null + if (byteLen < 0) return null + if (source.size < byteLen) return null + if (byteLen == 0) return null + + val limit = pw_decoder_push_limit(raw, byteLen) + + val result = mutableListOf() + + while (pw_decoder_bytes_until_limit(raw) > 0) { + val elem = readFn() ?: return null + result.add(elem) + } + + pw_decoder_pop_limit(raw, limit) + return result + } + /* * Based on the length of the packed repeated field, one of two list strategies is chosen. * If the length is less or equal a specific threshold (MAX_PACKED_BULK_SIZE), diff --git a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.native.kt b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.native.kt index f4cda86d3..29b7ca48f 100644 --- a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.native.kt +++ b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.native.kt @@ -11,8 +11,6 @@ import kotlin.experimental.ExperimentalNativeApi import kotlin.native.ref.createCleaner -// TODO: Evaluate if we should implement a ZeroCopyOutputSink (similar to the ZeroCopyInputSource) -// to reduce the number of copies during encoding. @OptIn(ExperimentalForeignApi::class, ExperimentalNativeApi::class) internal class WireEncoderNative(private val sink: Sink) : WireEncoder { /** @@ -125,39 +123,61 @@ internal class WireEncoderNative(private val sink: Sink) : WireEncoder { } } + override fun writePackedBool(fieldNr: Int, value: List, fieldSize: Int) = + writePackedInternal(fieldNr, value, fieldSize, ::pw_encoder_write_bool_no_tag) + + override fun writePackedInt32(fieldNr: Int, value: List, fieldSize: Int) = + writePackedInternal(fieldNr, value, fieldSize, ::pw_encoder_write_int32_no_tag) + + override fun writePackedInt64(fieldNr: Int, value: List, fieldSize: Int) = + writePackedInternal(fieldNr, value, fieldSize, ::pw_encoder_write_int64_no_tag) + + override fun writePackedUInt32(fieldNr: Int, value: List, fieldSize: Int) = + writePackedInternal(fieldNr, value, fieldSize, ::pw_encoder_write_uint32_no_tag) + + override fun writePackedUInt64(fieldNr: Int, value: List, fieldSize: Int) = + writePackedInternal(fieldNr, value, fieldSize, ::pw_encoder_write_uint64_no_tag) + + override fun writePackedSInt32(fieldNr: Int, value: List, fieldSize: Int) = + writePackedInternal(fieldNr, value, fieldSize, ::pw_encoder_write_sint32_no_tag) + + override fun writePackedSInt64(fieldNr: Int, value: List, fieldSize: Int) = + writePackedInternal(fieldNr, value, fieldSize, ::pw_encoder_write_sint64_no_tag) + override fun writePackedFixed32(fieldNr: Int, value: List) = - writePackedInternal(fieldNr, value, UInt.SIZE_BYTES, ::pw_encoder_write_fixed32_no_tag) + writePackedInternal(fieldNr, value, value.size * UInt.SIZE_BYTES, ::pw_encoder_write_fixed32_no_tag) override fun writePackedFixed64(fieldNr: Int, value: List) = - writePackedInternal(fieldNr, value, ULong.SIZE_BYTES, ::pw_encoder_write_fixed64_no_tag) + writePackedInternal(fieldNr, value, value.size * ULong.SIZE_BYTES, ::pw_encoder_write_fixed64_no_tag) override fun writePackedSFixed32(fieldNr: Int, value: List) = - writePackedInternal(fieldNr, value, Int.SIZE_BYTES, ::pw_encoder_write_sfixed32_no_tag) + writePackedInternal(fieldNr, value, value.size * Int.SIZE_BYTES, ::pw_encoder_write_sfixed32_no_tag) override fun writePackedSFixed64(fieldNr: Int, value: List) = - writePackedInternal(fieldNr, value, Long.SIZE_BYTES, ::pw_encoder_write_sfixed64_no_tag) + writePackedInternal(fieldNr, value, value.size * Long.SIZE_BYTES, ::pw_encoder_write_sfixed64_no_tag) override fun writePackedFloat(fieldNr: Int, value: List) = - writePackedInternal(fieldNr, value, Float.SIZE_BYTES, ::pw_encoder_write_float_no_tag) + writePackedInternal(fieldNr, value, value.size * Float.SIZE_BYTES, ::pw_encoder_write_float_no_tag) override fun writePackedDouble(fieldNr: Int, value: List) = - writePackedInternal(fieldNr, value, Double.SIZE_BYTES, ::pw_encoder_write_double_no_tag) + writePackedInternal(fieldNr, value, value.size * Double.SIZE_BYTES, ::pw_encoder_write_double_no_tag) } internal actual fun WireEncoder(sink: Sink): WireEncoder = WireEncoderNative(sink) +// the current implementation is slow, as it iterates through the list, to write each element individually, +// which can be speed up in case of fixed sized types, that are not compressed. KRPC-183 @OptIn(ExperimentalForeignApi::class) private inline fun WireEncoderNative.writePackedInternal( fieldNr: Int, value: List, - byteSize: Int, + fieldSize: Int, crossinline writer: (CValuesRef?, T) -> Boolean ): Boolean { val ktag = KTag(fieldNr, WireType.LENGTH_DELIMITED).toRawKTag() pw_encoder_write_tag(raw, ktag) // write the field size of the packed field - val fieldSize = value.size * byteSize; pw_encoder_write_int32_no_tag(raw, fieldSize) for (v in value) { if (!writer(raw, v)) { diff --git a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireSize.native.kt b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireSize.native.kt new file mode 100644 index 000000000..432479129 --- /dev/null +++ b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireSize.native.kt @@ -0,0 +1,18 @@ +/* + * Copyright 2023-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. + */ + +@file:OptIn(ExperimentalForeignApi::class) + +package kotlinx.rpc.grpc.internal + +import kotlinx.cinterop.ExperimentalForeignApi +import libprotowire.* + +internal actual fun WireSize.int32(value: Int) = pw_size_int32(value) +internal actual fun WireSize.int64(value: Long) = pw_size_int64(value) +internal actual fun WireSize.uInt32(value: UInt) = pw_size_uint32(value) +internal actual fun WireSize.uInt64(value: ULong) = pw_size_uint64(value) +internal actual fun WireSize.sInt32(value: Int) = pw_size_sint32(value) +internal actual fun WireSize.sInt64(value: Long) = pw_size_sint64(value) + diff --git a/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/WireCodecTest.kt b/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/WireCodecTest.kt index 01765c419..19bdf1333 100644 --- a/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/WireCodecTest.kt +++ b/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/WireCodecTest.kt @@ -664,7 +664,7 @@ class WireCodecTest { assertTrue(buffer.exhausted()) } - private fun runPackedTest( + private fun runPackedFixedTest( list: List, write: WireEncoder.(Int, List) -> Boolean, read: WireDecoder.() -> List?, @@ -686,44 +686,123 @@ class WireCodecTest { } @Test - fun testPackedFixed32() = runPackedTest( + fun testPackedFixed32() = runPackedFixedTest( List(1_000_000) { UInt.MAX_VALUE + it.toUInt() }, WireEncoder::writePackedFixed32, WireDecoder::readPackedFixed32 ) @Test - fun testPackedFixed64() = runPackedTest( + fun testPackedFixed64() = runPackedFixedTest( List(1_000_000) { UInt.MAX_VALUE + it.toULong() }, WireEncoder::writePackedFixed64, WireDecoder::readPackedFixed64, ) @Test - fun testPackedSFixed32() = runPackedTest( + fun testPackedSFixed32() = runPackedFixedTest( List(1_000_000) { Int.MAX_VALUE + it }, WireEncoder::writePackedSFixed32, WireDecoder::readPackedSFixed32 ) @Test - fun testPackedSFixed64() = runPackedTest( + fun testPackedSFixed64() = runPackedFixedTest( List(1_000_000) { Long.MAX_VALUE + it }, WireEncoder::writePackedSFixed64, WireDecoder::readPackedSFixed64 ) @Test - fun testPackedFloat() = runPackedTest( + fun testPackedFloat() = runPackedFixedTest( List(1_000_000) { it.toFloat() / 3.3f * ((it and 1) * 2 - 1) }, WireEncoder::writePackedFloat, WireDecoder::readPackedFloat, ) @Test - fun testPackedDouble() = runPackedTest( + fun testPackedDouble() = runPackedFixedTest( List(1_000_000) { it.toDouble() / 3.3 * ((it and 1) * 2 - 1) }, WireEncoder::writePackedDouble, WireDecoder::readPackedDouble, ) + + private fun runPackedVarTest( + list: List, + sizeFn: (List) -> UInt, + write: WireEncoder.(Int, List, Int) -> Boolean, + read: WireDecoder.() -> List?, + ) { + val buf = Buffer() + with(WireEncoder(buf)) { + assertTrue(write(1, list, sizeFn(list).toInt())) + flush() + } + WireDecoder(buf).use { dec -> + dec.readTag()!!.apply { + assertEquals(1, fieldNr) + assertEquals(WireType.LENGTH_DELIMITED, wireType) + } + val test = dec.read() + assertEquals(list, test) + } + assertTrue(buf.exhausted()) + } + + @Test + fun testPackedInt32() = runPackedVarTest( + List(1_000_000) { Int.MAX_VALUE + it }, + WireSize::packedInt32, + WireEncoder::writePackedInt32, + WireDecoder::readPackedInt32 + ) + + @Test + fun testPackedInt64() = runPackedVarTest( + List(1_000_000) { Long.MAX_VALUE + it.toLong() }, + WireSize::packedInt64, + WireEncoder::writePackedInt64, + WireDecoder::readPackedInt64 + ) + + @Test + fun testPackedUInt32() = runPackedVarTest( + List(1_000_000) { UInt.MAX_VALUE + it.toUInt() }, + WireSize::packedUInt32, + WireEncoder::writePackedUInt32, + WireDecoder::readPackedUInt32 + ) + + @Test + fun testPackedUInt64() = runPackedVarTest( + List(1_000_000) { ULong.MAX_VALUE + it.toULong() }, + WireSize::packedUInt64, + WireEncoder::writePackedUInt64, + WireDecoder::readPackedUInt64 + ) + + @Test + fun testPackedSInt32() = runPackedVarTest( + List(1_000_000) { Int.MAX_VALUE + it }, + WireSize::packedSInt32, + WireEncoder::writePackedSInt32, + WireDecoder::readPackedSInt32 + ) + + @Test + fun testPackedSInt64() = runPackedVarTest( + List(1_000_000) { Long.MAX_VALUE + it.toLong() }, + WireSize::packedSInt64, + WireEncoder::writePackedSInt64, + WireDecoder::readPackedSInt64 + ) + + @Test + fun testPackedEnum() = runPackedVarTest( + List(1_000_000) { it }, + WireSize::packedEnum, + WireEncoder::writePackedEnum, + WireDecoder::readPackedEnum + ) + } diff --git a/grpc/grpc-core/src/wasmJsMain/kotlin/kotlinx/rpc/grpc/internal/WireSize.wasmJs.kt b/grpc/grpc-core/src/wasmJsMain/kotlin/kotlinx/rpc/grpc/internal/WireSize.wasmJs.kt new file mode 100644 index 000000000..e70b9bd00 --- /dev/null +++ b/grpc/grpc-core/src/wasmJsMain/kotlin/kotlinx/rpc/grpc/internal/WireSize.wasmJs.kt @@ -0,0 +1,29 @@ +/* + * Copyright 2023-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.rpc.grpc.internal + +internal actual fun WireSize.int32(value: Int): UInt { + TODO("Not yet implemented") +} + +internal actual fun WireSize.int64(value: Long): UInt { + TODO("Not yet implemented") +} + +internal actual fun WireSize.uInt32(value: UInt): UInt { + TODO("Not yet implemented") +} + +internal actual fun WireSize.uInt64(value: ULong): UInt { + TODO("Not yet implemented") +} + +internal actual fun WireSize.sInt32(value: Int): UInt { + TODO("Not yet implemented") +} + +internal actual fun WireSize.sInt64(value: Long): UInt { + TODO("Not yet implemented") +} \ No newline at end of file diff --git a/grpc/grpcpp-c/include/protowire.h b/grpc/grpcpp-c/include/protowire.h index bcd12de64..1c62a7729 100644 --- a/grpc/grpcpp-c/include/protowire.h +++ b/grpc/grpcpp-c/include/protowire.h @@ -67,7 +67,6 @@ extern "C" { bool pw_encoder_write_float_no_tag(pw_encoder_t *self, float value); bool pw_encoder_write_double_no_tag(pw_encoder_t *self, double value); - //// WIRE DECODER //// typedef struct pw_decoder pw_decoder_t; @@ -115,6 +114,20 @@ extern "C" { // To read an actual bytes field, you must combine read_int32 and this function bool pw_decoder_read_raw_bytes(pw_decoder_t *self, void* buffer, int size); + int pw_decoder_push_limit(pw_decoder_t *self, int limit); + void pw_decoder_pop_limit(pw_decoder_t *self, int limit); + int pw_decoder_bytes_until_limit(pw_decoder_t *self); + + + /// Size Calculation Functions /// + + uint32_t pw_size_int32(int32_t value); + uint32_t pw_size_int64(int64_t value); + uint32_t pw_size_uint32(uint32_t value); + uint32_t pw_size_uint64(uint64_t value); + uint32_t pw_size_sint32(int32_t value); + uint32_t pw_size_sint64(int64_t value); + #ifdef __cplusplus } diff --git a/grpc/grpcpp-c/src/protowire.cpp b/grpc/grpcpp-c/src/protowire.cpp index 6f918842f..5614951ee 100644 --- a/grpc/grpcpp-c/src/protowire.cpp +++ b/grpc/grpcpp-c/src/protowire.cpp @@ -254,4 +254,36 @@ return check(self); \ bool pw_decoder_read_raw_bytes(pw_decoder_t *self, void* buffer, int size) { return self->cis.ReadRaw(buffer, size); } + + int pw_decoder_push_limit(pw_decoder_t *self, int limit) { + return self->cis.PushLimit(limit); + } + + void pw_decoder_pop_limit(pw_decoder_t *self, int limit) { + self->cis.PopLimit(limit); + } + + int pw_decoder_bytes_until_limit(pw_decoder_t *self) { + return self->cis.BytesUntilLimit(); + } + + uint32_t pw_size_int32(int32_t value) { + return WireFormatLite::Int32Size(value); + } + uint32_t pw_size_int64(int64_t value) { + return WireFormatLite::Int64Size(value); + } + uint32_t pw_size_uint32(uint32_t value) { + return WireFormatLite::UInt32Size(value); + } + uint32_t pw_size_uint64(uint64_t value) { + return WireFormatLite::UInt64Size(value); + } + uint32_t pw_size_sint32(int32_t value) { + return WireFormatLite::SInt32Size(value); + } + uint32_t pw_size_sint64(int64_t value) { + return WireFormatLite::SInt64Size(value); + } + } From a2893df121f9286cade9d94c17e693dfefefa881 Mon Sep 17 00:00:00 2001 From: Johannes Zottele Date: Mon, 28 Jul 2025 17:39:36 +0200 Subject: [PATCH 14/16] grpc-native: Address PR review issues Signed-off-by: Johannes Zottele --- .../kotlin/kotlinx/rpc/grpc/internal/KTag.kt | 2 +- .../kotlinx/rpc/grpc/internal/WireDecoder.kt | 6 + .../rpc/grpc/internal/WireDecoder.jvm.kt | 5 - .../kotlinx/rpc/grpc/internal/WireDecoder.kt | 21 - .../rpc/grpc/internal/WireDecoder.native.kt | 9 +- .../rpc/grpc/internal/WireEncoder.native.kt | 5 +- .../grpc/internal/bufferUnsafeExtensions.kt | 2 +- grpc/grpcpp-c/BUILD.bazel | 3 +- grpc/grpcpp-c/MODULE.bazel | 10 - grpc/grpcpp-c/MODULE.bazel.lock | 587 +----------------- grpc/grpcpp-c/include/protowire.h | 23 +- grpc/grpcpp-c/src/protowire.cpp | 106 ++-- 12 files changed, 93 insertions(+), 686 deletions(-) delete mode 100644 grpc/grpc-core/src/jvmMain/java/kotlinx/rpc/grpc/internal/WireDecoder.jvm.kt delete mode 100644 grpc/grpc-core/src/jvmMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.kt diff --git a/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/KTag.kt b/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/KTag.kt index 474ec4c36..cd9102250 100644 --- a/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/KTag.kt +++ b/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/KTag.kt @@ -34,7 +34,7 @@ internal fun KTag.toRawKTag(): UInt { return (fieldNr.toUInt() shl K_TAG_TYPE_BITS) or wireType.ordinal.toUInt() } -internal fun KTag.Companion.from(rawKTag: UInt): KTag? { +internal fun KTag.Companion.fromOrNull(rawKTag: UInt): KTag? { val type = (rawKTag and K_TAG_TYPE_MASK).toInt() val field = (rawKTag shr K_TAG_TYPE_BITS).toInt() if (!isValidFieldNr(field)) { diff --git a/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.kt b/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.kt index 84c2fd0c6..80c3e379d 100644 --- a/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.kt +++ b/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.kt @@ -9,6 +9,11 @@ import kotlinx.io.Buffer /** * A platform-specific decoder for wire format data. * + * This decoder is used by first calling [readTag], than looking up the field based on the field number in the returned, + * tag and then calling the actual `read*()` method to read the value to the corresponding field. + * This means that the nullable return value does not collide with optional fields, as optional fields would not + * include a tag in the encoded message. + * * If one `read*()` method returns `null`, decoding the data failed and no further * decoding can be done. * @@ -43,6 +48,7 @@ internal interface WireDecoder : AutoCloseable { fun readSFixed64(): Long? fun readFloat(): Float? fun readDouble(): Double? + fun readEnum(): Int? fun readString(): String? fun readBytes(): ByteArray? diff --git a/grpc/grpc-core/src/jvmMain/java/kotlinx/rpc/grpc/internal/WireDecoder.jvm.kt b/grpc/grpc-core/src/jvmMain/java/kotlinx/rpc/grpc/internal/WireDecoder.jvm.kt deleted file mode 100644 index 08a22f14b..000000000 --- a/grpc/grpc-core/src/jvmMain/java/kotlinx/rpc/grpc/internal/WireDecoder.jvm.kt +++ /dev/null @@ -1,5 +0,0 @@ -/* - * Copyright 2023-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. - */ - -package kotlinx.rpc.grpc.internal \ No newline at end of file diff --git a/grpc/grpc-core/src/jvmMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.kt b/grpc/grpc-core/src/jvmMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.kt deleted file mode 100644 index 63aa45c19..000000000 --- a/grpc/grpc-core/src/jvmMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.kt +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright 2023-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. - */ - -package kotlinx.rpc.grpc.internal - -import com.google.protobuf.CodedInputStream -import com.google.protobuf.CodedOutputStream - -@OptIn(ExperimentalUnsignedTypes::class) -internal class WireDecoder(val buffer: ByteArray) { - private val cos = CodedInputStream.newInstance(buffer) - - fun readTag(): Int { - return cos.readTag() - } - - fun readInt32(): Int { - return cos.readRawVarint32() - } -} \ No newline at end of file diff --git a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.native.kt b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.native.kt index af980e757..05e028711 100644 --- a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.native.kt +++ b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.native.kt @@ -12,6 +12,7 @@ import kotlin.experimental.ExperimentalNativeApi import kotlin.math.min import kotlin.native.ref.createCleaner +// TODO: Evaluate if this buffer size is suitable for all targets (KRPC-186) // maximum buffer size to allocate as contiguous memory in bytes private const val MAX_PACKED_BULK_SIZE: Int = 1_000_000 @@ -25,7 +26,7 @@ internal class WireDecoderNative(private val source: Buffer) : WireDecoder { // construct the pw_decoder_t by passing a pw_zero_copy_input_t that provides a bridge between // the CodedInputStream and the given source buffer. it passes functions that call the respective // ZeroCopyInputSource methods. - internal val raw = run { + internal val raw: CPointer = run { // construct the pw_zero_copy_input_t that functions as a bridge to the ZeroCopyInputSource val zeroCopyCInput = cValue { ctx = zeroCopyInput.asCPointer() @@ -62,7 +63,7 @@ internal class WireDecoderNative(private val source: Buffer) : WireDecoder { override fun readTag(): KTag? { val tag = pw_decoder_read_tag(raw) - return KTag.from(tag) + return KTag.fromOrNull(tag) } override fun readBool(): Boolean? = memScoped { @@ -177,7 +178,7 @@ internal class WireDecoderNative(private val source: Buffer) : WireDecoder { return null } - // TODO: Is it possible to avoid copying the c_str, by directly allocating a K/N String (as in readBytes)? + // TODO: Is it possible to avoid copying the c_str, by directly allocating a K/N String (as in readBytes)? KRPC-187 override fun readString(): String? = memScoped { val str = alloc>() val ok = pw_decoder_read_string(raw, str.ptr) @@ -263,7 +264,7 @@ internal class WireDecoderNative(private val source: Buffer) : WireDecoder { val byteLen = readInt32() ?: return null if (byteLen < 0) return null if (source.size < byteLen) return null - if (byteLen == 0) return null + if (byteLen == 0) return emptyList() val limit = pw_decoder_push_limit(raw, byteLen) diff --git a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.native.kt b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.native.kt index 29b7ca48f..9944ce0af 100644 --- a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.native.kt +++ b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.native.kt @@ -30,7 +30,7 @@ internal class WireEncoderNative(private val sink: Sink) : WireEncoder { private val context = StableRef.create(this.Ctx()) // construct encoder with a callback that calls write() on this.context - internal val raw = run { + internal val raw: CPointer = run { pw_encoder_new(context.asCPointer(), staticCFunction { ctx, buf, size -> if (buf == null || ctx == null) { return@staticCFunction false @@ -175,8 +175,7 @@ private inline fun WireEncoderNative.writePackedInternal( fieldSize: Int, crossinline writer: (CValuesRef?, T) -> Boolean ): Boolean { - val ktag = KTag(fieldNr, WireType.LENGTH_DELIMITED).toRawKTag() - pw_encoder_write_tag(raw, ktag) + pw_encoder_write_tag(raw, fieldNr, WireType.LENGTH_DELIMITED.ordinal) // write the field size of the packed field pw_encoder_write_int32_no_tag(raw, fieldSize) for (v in value) { diff --git a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/bufferUnsafeExtensions.kt b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/bufferUnsafeExtensions.kt index fdfcff4d3..f1e1495ae 100644 --- a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/bufferUnsafeExtensions.kt +++ b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/bufferUnsafeExtensions.kt @@ -13,7 +13,7 @@ import platform.posix.memcpy @OptIn(ExperimentalForeignApi::class, InternalIoApi::class, UnsafeIoApi::class) -public fun Sink.writeFully(buffer: CPointer, offset: Long, length: Long) { +internal fun Sink.writeFully(buffer: CPointer, offset: Long, length: Long) { var consumed = 0L while (consumed < length) { UnsafeBufferOperations.writeToTail(this.buffer, 1) { array, start, endExclusive -> diff --git a/grpc/grpcpp-c/BUILD.bazel b/grpc/grpcpp-c/BUILD.bazel index 566d973df..4e3157af4 100644 --- a/grpc/grpcpp-c/BUILD.bazel +++ b/grpc/grpcpp-c/BUILD.bazel @@ -1,4 +1,3 @@ -load("@emsdk//emscripten_toolchain:wasm_rules.bzl", "wasm_cc_binary") load("@rules_cc//cc:defs.bzl", "cc_library") cc_binary( @@ -24,7 +23,7 @@ cc_library( includes = ["include"], visibility = ["//visibility:public"], deps = [ - # TODO: Reduce the dependencies and only use required once + # TODO: Reduce the dependencies and only use required once. KRPC-185 "@com_github_grpc_grpc//:channelz", "@com_github_grpc_grpc//:generic_stub", "@com_github_grpc_grpc//:grpc++", diff --git a/grpc/grpcpp-c/MODULE.bazel b/grpc/grpcpp-c/MODULE.bazel index b729c8a44..136cad3ae 100644 --- a/grpc/grpcpp-c/MODULE.bazel +++ b/grpc/grpcpp-c/MODULE.bazel @@ -22,13 +22,3 @@ bazel_dep( version = "1.73.1", repo_name = "com_github_grpc_grpc", ) - -emsdk_version = "4.0.11" - -bazel_dep(name = "emsdk", version = emsdk_version) -git_override( - module_name = "emsdk", - remote = "https://github.com/emscripten-core/emsdk.git", - strip_prefix = "bazel", - tag = emsdk_version, -) diff --git a/grpc/grpcpp-c/MODULE.bazel.lock b/grpc/grpcpp-c/MODULE.bazel.lock index 9e94f9a39..09e36896c 100644 --- a/grpc/grpcpp-c/MODULE.bazel.lock +++ b/grpc/grpcpp-c/MODULE.bazel.lock @@ -24,12 +24,8 @@ "https://bcr.bazel.build/modules/aspect_bazel_lib/1.31.2/MODULE.bazel": "7bee702b4862612f29333590f4b658a5832d433d6f8e4395f090e8f4e85d442f", "https://bcr.bazel.build/modules/aspect_bazel_lib/1.38.0/MODULE.bazel": "6307fec451ba9962c1c969eb516ebfe1e46528f7fa92e1c9ac8646bef4cdaa3f", "https://bcr.bazel.build/modules/aspect_bazel_lib/1.40.3/MODULE.bazel": "668e6bcb4d957fc0e284316dba546b705c8d43c857f87119619ee83c4555b859", - "https://bcr.bazel.build/modules/aspect_bazel_lib/1.42.3/MODULE.bazel": "e4529e12d8cd5b828e2b5960d07d3ec032541740d419d7d5b859cabbf5b056f9", - "https://bcr.bazel.build/modules/aspect_bazel_lib/1.42.3/source.json": "80cb66069ad626e0921555cd2bf278286fd7763fae2450e564e351792e8303f4", "https://bcr.bazel.build/modules/aspect_rules_js/1.33.1/MODULE.bazel": "db3e7f16e471cf6827059d03af7c21859e7a0d2bc65429a3a11f005d46fc501b", "https://bcr.bazel.build/modules/aspect_rules_js/1.39.0/MODULE.bazel": "aece421d479e3c31dc3e5f6d49a12acc2700457c03c556650ec7a0ff23fc0d95", - "https://bcr.bazel.build/modules/aspect_rules_js/1.42.0/MODULE.bazel": "f19e6b4a16f77f8cf3728eac1f60dbfd8e043517fd4f4dbf17a75a6c50936d62", - "https://bcr.bazel.build/modules/aspect_rules_js/1.42.0/source.json": "abbb3eac3b6af76b8ce230a9a901c6d08d93f4f5ffd55314bf630827dddee57e", "https://bcr.bazel.build/modules/aspect_rules_lint/0.12.0/MODULE.bazel": "e767c5dbfeb254ec03275a7701b5cfde2c4d2873676804bc7cb27ddff3728fed", "https://bcr.bazel.build/modules/bazel_features/0.1.0/MODULE.bazel": "47011d645b0f949f42ee67f2e8775188a9cf4a0a1528aa2fa4952f2fd00906fd", "https://bcr.bazel.build/modules/bazel_features/1.1.0/MODULE.bazel": "cfd42ff3b815a5f39554d97182657f8c4b9719568eb7fded2b9135f084bf760b", @@ -46,7 +42,6 @@ "https://bcr.bazel.build/modules/bazel_features/1.30.0/MODULE.bazel": "a14b62d05969a293b80257e72e597c2da7f717e1e69fa8b339703ed6731bec87", "https://bcr.bazel.build/modules/bazel_features/1.30.0/source.json": "b07e17f067fe4f69f90b03b36ef1e08fe0d1f3cac254c1241a1818773e3423bc", "https://bcr.bazel.build/modules/bazel_features/1.4.1/MODULE.bazel": "e45b6bb2350aff3e442ae1111c555e27eac1d915e77775f6fdc4b351b758b5d7", - "https://bcr.bazel.build/modules/bazel_features/1.9.0/MODULE.bazel": "885151d58d90d8d9c811eb75e3288c11f850e1d6b481a8c9f766adee4712358b", "https://bcr.bazel.build/modules/bazel_features/1.9.1/MODULE.bazel": "8f679097876a9b609ad1f60249c49d68bfab783dd9be012faf9d82547b14815a", "https://bcr.bazel.build/modules/bazel_skylib/1.0.3/MODULE.bazel": "bcb0fd896384802d1ad283b4e4eb4d718eebd8cb820b0a2c3a347fb971afd9d8", "https://bcr.bazel.build/modules/bazel_skylib/1.1.1/MODULE.bazel": "1add3e7d93ff2e6998f9e118022c84d163917d912f5afafb3058e3d2f1545b5e", @@ -259,8 +254,6 @@ "https://bcr.bazel.build/modules/rules_license/1.0.0/MODULE.bazel": "a7fda60eefdf3d8c827262ba499957e4df06f659330bbe6cdbdb975b768bb65c", "https://bcr.bazel.build/modules/rules_license/1.0.0/source.json": "a52c89e54cc311196e478f8382df91c15f7a2bfdf4c6cd0e2675cc2ff0b56efb", "https://bcr.bazel.build/modules/rules_nodejs/5.8.2/MODULE.bazel": "6bc03c8f37f69401b888023bf511cb6ee4781433b0cb56236b2e55a21e3a026a", - "https://bcr.bazel.build/modules/rules_nodejs/6.3.2/MODULE.bazel": "42e8d5254b6135f890fecca7c8d7f95a7d27a45f8275b276f66ec337767530ef", - "https://bcr.bazel.build/modules/rules_nodejs/6.3.2/source.json": "80e0a68eb81772f1631f8b69014884eebc2474b3b3025fd19a5240ae4f76f9c9", "https://bcr.bazel.build/modules/rules_perl/0.2.4/MODULE.bazel": "5f5af7be4bf5fb88d91af7469518f0fd2161718aefc606188f7cd51f436ca938", "https://bcr.bazel.build/modules/rules_perl/0.2.4/source.json": "574317d6b3c7e4843fe611b76f15e62a1889949f5570702e1ee4ad335ea3c339", "https://bcr.bazel.build/modules/rules_pkg/0.7.0/MODULE.bazel": "df99f03fc7934a4737122518bb87e667e62d780b610910f0447665a7e2be62dc", @@ -287,8 +280,7 @@ "https://bcr.bazel.build/modules/rules_python/0.4.0/MODULE.bazel": "9208ee05fd48bf09ac60ed269791cf17fb343db56c8226a720fbb1cdf467166c", "https://bcr.bazel.build/modules/rules_python/0.40.0/MODULE.bazel": "9d1a3cd88ed7d8e39583d9ffe56ae8a244f67783ae89b60caafc9f5cf318ada7", "https://bcr.bazel.build/modules/rules_python/1.0.0/MODULE.bazel": "898a3d999c22caa585eb062b600f88654bf92efb204fa346fb55f6f8edffca43", - "https://bcr.bazel.build/modules/rules_python/1.3.0/MODULE.bazel": "8361d57eafb67c09b75bf4bbe6be360e1b8f4f18118ab48037f2bd50aa2ccb13", - "https://bcr.bazel.build/modules/rules_python/1.3.0/source.json": "25932f917cd279c7baefa6cb1d3fa8750a7a29de522024449b19af6eab51f4a0", + "https://bcr.bazel.build/modules/rules_python/1.0.0/source.json": "b0162a65c6312e45e7912e39abd1a7f8856c2c7e41ecc9b6dc688a6f6400a917", "https://bcr.bazel.build/modules/rules_rust/0.51.0/MODULE.bazel": "2b6d1617ac8503bfdcc0e4520c20539d4bba3a691100bee01afe193ceb0310f9", "https://bcr.bazel.build/modules/rules_shell/0.2.0/MODULE.bazel": "fda8a652ab3c7d8fee214de05e7a9916d8b28082234e8d2c0094505c5268ed3c", "https://bcr.bazel.build/modules/rules_shell/0.3.0/MODULE.bazel": "de4402cd12f4cc8fda2354fce179fdb068c0b9ca1ec2d2b17b3e21b24c1a937b", @@ -358,426 +350,6 @@ ] } }, - "@@aspect_bazel_lib+//lib:extensions.bzl%toolchains": { - "general": { - "bzlTransitiveDigest": "FRH/uLcAIxs4VtonQvNHnu3yGF1glDBtm+FyaO6lOI4=", - "usagesDigest": "1c7PNX163TGNqWzfejRnWpH/hiT4/GRG0kYxuez0Uz0=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "copy_directory_darwin_amd64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:copy_directory_toolchain.bzl%copy_directory_platform_repo", - "attributes": { - "platform": "darwin_amd64" - } - }, - "copy_directory_darwin_arm64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:copy_directory_toolchain.bzl%copy_directory_platform_repo", - "attributes": { - "platform": "darwin_arm64" - } - }, - "copy_directory_freebsd_amd64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:copy_directory_toolchain.bzl%copy_directory_platform_repo", - "attributes": { - "platform": "freebsd_amd64" - } - }, - "copy_directory_linux_amd64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:copy_directory_toolchain.bzl%copy_directory_platform_repo", - "attributes": { - "platform": "linux_amd64" - } - }, - "copy_directory_linux_arm64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:copy_directory_toolchain.bzl%copy_directory_platform_repo", - "attributes": { - "platform": "linux_arm64" - } - }, - "copy_directory_windows_amd64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:copy_directory_toolchain.bzl%copy_directory_platform_repo", - "attributes": { - "platform": "windows_amd64" - } - }, - "copy_directory_toolchains": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:copy_directory_toolchain.bzl%copy_directory_toolchains_repo", - "attributes": { - "user_repository_name": "copy_directory" - } - }, - "copy_to_directory_darwin_amd64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:copy_to_directory_toolchain.bzl%copy_to_directory_platform_repo", - "attributes": { - "platform": "darwin_amd64" - } - }, - "copy_to_directory_darwin_arm64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:copy_to_directory_toolchain.bzl%copy_to_directory_platform_repo", - "attributes": { - "platform": "darwin_arm64" - } - }, - "copy_to_directory_freebsd_amd64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:copy_to_directory_toolchain.bzl%copy_to_directory_platform_repo", - "attributes": { - "platform": "freebsd_amd64" - } - }, - "copy_to_directory_linux_amd64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:copy_to_directory_toolchain.bzl%copy_to_directory_platform_repo", - "attributes": { - "platform": "linux_amd64" - } - }, - "copy_to_directory_linux_arm64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:copy_to_directory_toolchain.bzl%copy_to_directory_platform_repo", - "attributes": { - "platform": "linux_arm64" - } - }, - "copy_to_directory_windows_amd64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:copy_to_directory_toolchain.bzl%copy_to_directory_platform_repo", - "attributes": { - "platform": "windows_amd64" - } - }, - "copy_to_directory_toolchains": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:copy_to_directory_toolchain.bzl%copy_to_directory_toolchains_repo", - "attributes": { - "user_repository_name": "copy_to_directory" - } - }, - "jq_darwin_amd64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:jq_toolchain.bzl%jq_platform_repo", - "attributes": { - "platform": "darwin_amd64", - "version": "1.6" - } - }, - "jq_darwin_arm64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:jq_toolchain.bzl%jq_platform_repo", - "attributes": { - "platform": "darwin_arm64", - "version": "1.6" - } - }, - "jq_linux_amd64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:jq_toolchain.bzl%jq_platform_repo", - "attributes": { - "platform": "linux_amd64", - "version": "1.6" - } - }, - "jq_windows_amd64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:jq_toolchain.bzl%jq_platform_repo", - "attributes": { - "platform": "windows_amd64", - "version": "1.6" - } - }, - "jq": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:jq_toolchain.bzl%jq_host_alias_repo", - "attributes": {} - }, - "jq_toolchains": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:jq_toolchain.bzl%jq_toolchains_repo", - "attributes": { - "user_repository_name": "jq" - } - }, - "yq_darwin_amd64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:yq_toolchain.bzl%yq_platform_repo", - "attributes": { - "platform": "darwin_amd64", - "version": "4.25.2" - } - }, - "yq_darwin_arm64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:yq_toolchain.bzl%yq_platform_repo", - "attributes": { - "platform": "darwin_arm64", - "version": "4.25.2" - } - }, - "yq_linux_amd64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:yq_toolchain.bzl%yq_platform_repo", - "attributes": { - "platform": "linux_amd64", - "version": "4.25.2" - } - }, - "yq_linux_arm64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:yq_toolchain.bzl%yq_platform_repo", - "attributes": { - "platform": "linux_arm64", - "version": "4.25.2" - } - }, - "yq_linux_s390x": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:yq_toolchain.bzl%yq_platform_repo", - "attributes": { - "platform": "linux_s390x", - "version": "4.25.2" - } - }, - "yq_linux_ppc64le": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:yq_toolchain.bzl%yq_platform_repo", - "attributes": { - "platform": "linux_ppc64le", - "version": "4.25.2" - } - }, - "yq_windows_amd64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:yq_toolchain.bzl%yq_platform_repo", - "attributes": { - "platform": "windows_amd64", - "version": "4.25.2" - } - }, - "yq": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:yq_toolchain.bzl%yq_host_alias_repo", - "attributes": {} - }, - "yq_toolchains": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:yq_toolchain.bzl%yq_toolchains_repo", - "attributes": { - "user_repository_name": "yq" - } - }, - "coreutils_darwin_amd64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:coreutils_toolchain.bzl%coreutils_platform_repo", - "attributes": { - "platform": "darwin_amd64", - "version": "0.0.16" - } - }, - "coreutils_darwin_arm64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:coreutils_toolchain.bzl%coreutils_platform_repo", - "attributes": { - "platform": "darwin_arm64", - "version": "0.0.16" - } - }, - "coreutils_linux_amd64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:coreutils_toolchain.bzl%coreutils_platform_repo", - "attributes": { - "platform": "linux_amd64", - "version": "0.0.16" - } - }, - "coreutils_linux_arm64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:coreutils_toolchain.bzl%coreutils_platform_repo", - "attributes": { - "platform": "linux_arm64", - "version": "0.0.16" - } - }, - "coreutils_windows_amd64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:coreutils_toolchain.bzl%coreutils_platform_repo", - "attributes": { - "platform": "windows_amd64", - "version": "0.0.16" - } - }, - "coreutils_toolchains": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:coreutils_toolchain.bzl%coreutils_toolchains_repo", - "attributes": { - "user_repository_name": "coreutils" - } - }, - "bsd_tar_darwin_amd64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:tar_toolchain.bzl%bsdtar_binary_repo", - "attributes": { - "platform": "darwin_amd64" - } - }, - "bsd_tar_darwin_arm64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:tar_toolchain.bzl%bsdtar_binary_repo", - "attributes": { - "platform": "darwin_arm64" - } - }, - "bsd_tar_linux_amd64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:tar_toolchain.bzl%bsdtar_binary_repo", - "attributes": { - "platform": "linux_amd64" - } - }, - "bsd_tar_linux_arm64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:tar_toolchain.bzl%bsdtar_binary_repo", - "attributes": { - "platform": "linux_arm64" - } - }, - "bsd_tar_windows_amd64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:tar_toolchain.bzl%bsdtar_binary_repo", - "attributes": { - "platform": "windows_amd64" - } - }, - "bsd_tar_toolchains": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:tar_toolchain.bzl%tar_toolchains_repo", - "attributes": { - "user_repository_name": "bsd_tar" - } - }, - "expand_template_darwin_amd64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:expand_template_toolchain.bzl%expand_template_platform_repo", - "attributes": { - "platform": "darwin_amd64" - } - }, - "expand_template_darwin_arm64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:expand_template_toolchain.bzl%expand_template_platform_repo", - "attributes": { - "platform": "darwin_arm64" - } - }, - "expand_template_freebsd_amd64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:expand_template_toolchain.bzl%expand_template_platform_repo", - "attributes": { - "platform": "freebsd_amd64" - } - }, - "expand_template_linux_amd64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:expand_template_toolchain.bzl%expand_template_platform_repo", - "attributes": { - "platform": "linux_amd64" - } - }, - "expand_template_linux_arm64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:expand_template_toolchain.bzl%expand_template_platform_repo", - "attributes": { - "platform": "linux_arm64" - } - }, - "expand_template_windows_amd64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:expand_template_toolchain.bzl%expand_template_platform_repo", - "attributes": { - "platform": "windows_amd64" - } - }, - "expand_template_toolchains": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:expand_template_toolchain.bzl%expand_template_toolchains_repo", - "attributes": { - "user_repository_name": "expand_template" - } - } - }, - "recordedRepoMappingEntries": [ - [ - "aspect_bazel_lib+", - "aspect_bazel_lib", - "aspect_bazel_lib+" - ], - [ - "aspect_bazel_lib+", - "bazel_skylib", - "bazel_skylib+" - ], - [ - "aspect_bazel_lib+", - "bazel_tools", - "bazel_tools" - ] - ] - } - }, - "@@emsdk+//:emscripten_cache.bzl%emscripten_cache": { - "general": { - "bzlTransitiveDigest": "uqDvXmpTNqW4+ie/Fk+xC3TrFrKvL+9hNtoP51Kt2oo=", - "usagesDigest": "iaw2BH+XNky0wzaCMCWcOxr/wRXVypwc4a22UDwIjIs=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "emscripten_cache": { - "repoRuleId": "@@emsdk+//:emscripten_cache.bzl%_emscripten_cache_repository", - "attributes": { - "configuration": [], - "targets": [] - } - } - }, - "recordedRepoMappingEntries": [] - } - }, - "@@emsdk+//:emscripten_deps.bzl%emscripten_deps": { - "general": { - "bzlTransitiveDigest": "hjVXd1Th8cZxk9rw2px6WtJqoM900gx6e4EEMalHmis=", - "usagesDigest": "hZ+VngAMPf2nklmauHosp51Gh/R1WN3gtTJ/lVPCmjg=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "emscripten_bin_linux": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "build_file_content": "\npackage(default_visibility = ['//visibility:public'])\n\nfilegroup(\n name = \"all\",\n srcs = glob([\"**\"]),\n)\n\nfilegroup(\n name = \"includes\",\n srcs = glob([\n \"emscripten/cache/sysroot/include/c++/v1/**\",\n \"emscripten/cache/sysroot/include/compat/**\",\n \"emscripten/cache/sysroot/include/**\",\n \"lib/clang/**/include/**\",\n ]),\n)\n\nfilegroup(\n name = \"emcc_common\",\n srcs = [\n \"emscripten/emcc.py\",\n \"emscripten/embuilder.py\",\n \"emscripten/emscripten-version.txt\",\n \"emscripten/cache/sysroot_install.stamp\",\n \"emscripten/src/settings.js\",\n \"emscripten/src/settings_internal.js\",\n ] + glob(\n include = [\n \"emscripten/third_party/**\",\n \"emscripten/tools/**\",\n ],\n exclude = [\n \"**/__pycache__/**\",\n ],\n ),\n)\n\nfilegroup(\n name = \"compiler_files\",\n srcs = [\n \"bin/clang\",\n \"bin/clang++\",\n \":emcc_common\",\n \":includes\",\n ],\n)\n\nfilegroup(\n name = \"linker_files\",\n srcs = [\n \"bin/clang\",\n \"bin/llvm-ar\",\n \"bin/llvm-dwarfdump\",\n \"bin/llvm-nm\",\n \"bin/llvm-objcopy\",\n \"bin/wasm-ctor-eval\",\n \"bin/wasm-emscripten-finalize\",\n \"bin/wasm-ld\",\n \"bin/wasm-metadce\",\n \"bin/wasm-opt\",\n \"bin/wasm-split\",\n \"bin/wasm2js\",\n \":emcc_common\",\n ] + glob(\n include = [\n \"emscripten/cache/sysroot/lib/**\",\n \"emscripten/node_modules/**\",\n \"emscripten/src/**\",\n ],\n ),\n)\n\nfilegroup(\n name = \"ar_files\",\n srcs = [\n \"bin/llvm-ar\",\n \"emscripten/emar.py\",\n \"emscripten/emscripten-version.txt\",\n \"emscripten/src/settings.js\",\n \"emscripten/src/settings_internal.js\",\n ] + glob(\n include = [\n \"emscripten/tools/**\",\n ],\n exclude = [\n \"**/__pycache__/**\",\n ],\n ),\n)\n", - "sha256": "f38e70b53be587e7c757f375b3452e259c70130d4b40db3213c95b7ae321f5d7", - "strip_prefix": "install", - "type": "tar.xz", - "url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/linux/7033fec38817ec01909b044ea0193ddd5057255c/wasm-binaries.tar.xz" - } - }, - "emscripten_bin_linux_arm64": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "build_file_content": "\npackage(default_visibility = ['//visibility:public'])\n\nfilegroup(\n name = \"all\",\n srcs = glob([\"**\"]),\n)\n\nfilegroup(\n name = \"includes\",\n srcs = glob([\n \"emscripten/cache/sysroot/include/c++/v1/**\",\n \"emscripten/cache/sysroot/include/compat/**\",\n \"emscripten/cache/sysroot/include/**\",\n \"lib/clang/**/include/**\",\n ]),\n)\n\nfilegroup(\n name = \"emcc_common\",\n srcs = [\n \"emscripten/emcc.py\",\n \"emscripten/embuilder.py\",\n \"emscripten/emscripten-version.txt\",\n \"emscripten/cache/sysroot_install.stamp\",\n \"emscripten/src/settings.js\",\n \"emscripten/src/settings_internal.js\",\n ] + glob(\n include = [\n \"emscripten/third_party/**\",\n \"emscripten/tools/**\",\n ],\n exclude = [\n \"**/__pycache__/**\",\n ],\n ),\n)\n\nfilegroup(\n name = \"compiler_files\",\n srcs = [\n \"bin/clang\",\n \"bin/clang++\",\n \":emcc_common\",\n \":includes\",\n ],\n)\n\nfilegroup(\n name = \"linker_files\",\n srcs = [\n \"bin/clang\",\n \"bin/llvm-ar\",\n \"bin/llvm-dwarfdump\",\n \"bin/llvm-nm\",\n \"bin/llvm-objcopy\",\n \"bin/wasm-ctor-eval\",\n \"bin/wasm-emscripten-finalize\",\n \"bin/wasm-ld\",\n \"bin/wasm-metadce\",\n \"bin/wasm-opt\",\n \"bin/wasm-split\",\n \"bin/wasm2js\",\n \":emcc_common\",\n ] + glob(\n include = [\n \"emscripten/cache/sysroot/lib/**\",\n \"emscripten/node_modules/**\",\n \"emscripten/src/**\",\n ],\n ),\n)\n\nfilegroup(\n name = \"ar_files\",\n srcs = [\n \"bin/llvm-ar\",\n \"emscripten/emar.py\",\n \"emscripten/emscripten-version.txt\",\n \"emscripten/src/settings.js\",\n \"emscripten/src/settings_internal.js\",\n ] + glob(\n include = [\n \"emscripten/tools/**\",\n ],\n exclude = [\n \"**/__pycache__/**\",\n ],\n ),\n)\n", - "sha256": "42020e4db200ac366a3e91ac2fccc04ee0ffc090cd2d5986c892b27f39172bb9", - "strip_prefix": "install", - "type": "tar.xz", - "url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/linux/7033fec38817ec01909b044ea0193ddd5057255c/wasm-binaries-arm64.tar.xz" - } - }, - "emscripten_bin_mac": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "build_file_content": "\npackage(default_visibility = ['//visibility:public'])\n\nfilegroup(\n name = \"all\",\n srcs = glob([\"**\"]),\n)\n\nfilegroup(\n name = \"includes\",\n srcs = glob([\n \"emscripten/cache/sysroot/include/c++/v1/**\",\n \"emscripten/cache/sysroot/include/compat/**\",\n \"emscripten/cache/sysroot/include/**\",\n \"lib/clang/**/include/**\",\n ]),\n)\n\nfilegroup(\n name = \"emcc_common\",\n srcs = [\n \"emscripten/emcc.py\",\n \"emscripten/embuilder.py\",\n \"emscripten/emscripten-version.txt\",\n \"emscripten/cache/sysroot_install.stamp\",\n \"emscripten/src/settings.js\",\n \"emscripten/src/settings_internal.js\",\n ] + glob(\n include = [\n \"emscripten/third_party/**\",\n \"emscripten/tools/**\",\n ],\n exclude = [\n \"**/__pycache__/**\",\n ],\n ),\n)\n\nfilegroup(\n name = \"compiler_files\",\n srcs = [\n \"bin/clang\",\n \"bin/clang++\",\n \":emcc_common\",\n \":includes\",\n ],\n)\n\nfilegroup(\n name = \"linker_files\",\n srcs = [\n \"bin/clang\",\n \"bin/llvm-ar\",\n \"bin/llvm-dwarfdump\",\n \"bin/llvm-nm\",\n \"bin/llvm-objcopy\",\n \"bin/wasm-ctor-eval\",\n \"bin/wasm-emscripten-finalize\",\n \"bin/wasm-ld\",\n \"bin/wasm-metadce\",\n \"bin/wasm-opt\",\n \"bin/wasm-split\",\n \"bin/wasm2js\",\n \":emcc_common\",\n ] + glob(\n include = [\n \"emscripten/cache/sysroot/lib/**\",\n \"emscripten/node_modules/**\",\n \"emscripten/src/**\",\n ],\n ),\n)\n\nfilegroup(\n name = \"ar_files\",\n srcs = [\n \"bin/llvm-ar\",\n \"emscripten/emar.py\",\n \"emscripten/emscripten-version.txt\",\n \"emscripten/src/settings.js\",\n \"emscripten/src/settings_internal.js\",\n ] + glob(\n include = [\n \"emscripten/tools/**\",\n ],\n exclude = [\n \"**/__pycache__/**\",\n ],\n ),\n)\n", - "sha256": "4169811f9682f54ae5c9d0662d0a4dd4318abab5d3d0473fa54007f515a8cdea", - "strip_prefix": "install", - "type": "tar.xz", - "url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/mac/7033fec38817ec01909b044ea0193ddd5057255c/wasm-binaries.tar.xz" - } - }, - "emscripten_bin_mac_arm64": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "build_file_content": "\npackage(default_visibility = ['//visibility:public'])\n\nfilegroup(\n name = \"all\",\n srcs = glob([\"**\"]),\n)\n\nfilegroup(\n name = \"includes\",\n srcs = glob([\n \"emscripten/cache/sysroot/include/c++/v1/**\",\n \"emscripten/cache/sysroot/include/compat/**\",\n \"emscripten/cache/sysroot/include/**\",\n \"lib/clang/**/include/**\",\n ]),\n)\n\nfilegroup(\n name = \"emcc_common\",\n srcs = [\n \"emscripten/emcc.py\",\n \"emscripten/embuilder.py\",\n \"emscripten/emscripten-version.txt\",\n \"emscripten/cache/sysroot_install.stamp\",\n \"emscripten/src/settings.js\",\n \"emscripten/src/settings_internal.js\",\n ] + glob(\n include = [\n \"emscripten/third_party/**\",\n \"emscripten/tools/**\",\n ],\n exclude = [\n \"**/__pycache__/**\",\n ],\n ),\n)\n\nfilegroup(\n name = \"compiler_files\",\n srcs = [\n \"bin/clang\",\n \"bin/clang++\",\n \":emcc_common\",\n \":includes\",\n ],\n)\n\nfilegroup(\n name = \"linker_files\",\n srcs = [\n \"bin/clang\",\n \"bin/llvm-ar\",\n \"bin/llvm-dwarfdump\",\n \"bin/llvm-nm\",\n \"bin/llvm-objcopy\",\n \"bin/wasm-ctor-eval\",\n \"bin/wasm-emscripten-finalize\",\n \"bin/wasm-ld\",\n \"bin/wasm-metadce\",\n \"bin/wasm-opt\",\n \"bin/wasm-split\",\n \"bin/wasm2js\",\n \":emcc_common\",\n ] + glob(\n include = [\n \"emscripten/cache/sysroot/lib/**\",\n \"emscripten/node_modules/**\",\n \"emscripten/src/**\",\n ],\n ),\n)\n\nfilegroup(\n name = \"ar_files\",\n srcs = [\n \"bin/llvm-ar\",\n \"emscripten/emar.py\",\n \"emscripten/emscripten-version.txt\",\n \"emscripten/src/settings.js\",\n \"emscripten/src/settings_internal.js\",\n ] + glob(\n include = [\n \"emscripten/tools/**\",\n ],\n exclude = [\n \"**/__pycache__/**\",\n ],\n ),\n)\n", - "sha256": "09554371e3941306d047d67618532e5366ba6c9b5bda1a504a917bfbabc5d414", - "strip_prefix": "install", - "type": "tar.xz", - "url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/mac/7033fec38817ec01909b044ea0193ddd5057255c/wasm-binaries-arm64.tar.xz" - } - }, - "emscripten_bin_win": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "build_file_content": "\npackage(default_visibility = ['//visibility:public'])\n\nfilegroup(\n name = \"all\",\n srcs = glob([\"**\"]),\n)\n\nfilegroup(\n name = \"includes\",\n srcs = glob([\n \"emscripten/cache/sysroot/include/c++/v1/**\",\n \"emscripten/cache/sysroot/include/compat/**\",\n \"emscripten/cache/sysroot/include/**\",\n \"lib/clang/**/include/**\",\n ]),\n)\n\nfilegroup(\n name = \"emcc_common\",\n srcs = [\n \"emscripten/emcc.py\",\n \"emscripten/embuilder.py\",\n \"emscripten/emscripten-version.txt\",\n \"emscripten/cache/sysroot_install.stamp\",\n \"emscripten/src/settings.js\",\n \"emscripten/src/settings_internal.js\",\n ] + glob(\n include = [\n \"emscripten/third_party/**\",\n \"emscripten/tools/**\",\n ],\n exclude = [\n \"**/__pycache__/**\",\n ],\n ),\n)\n\nfilegroup(\n name = \"compiler_files\",\n srcs = [\n \"bin/clang.exe\",\n \"bin/clang++.exe\",\n \":emcc_common\",\n \":includes\",\n ],\n)\n\nfilegroup(\n name = \"linker_files\",\n srcs = [\n \"bin/clang.exe\",\n \"bin/llvm-ar.exe\",\n \"bin/llvm-dwarfdump.exe\",\n \"bin/llvm-nm.exe\",\n \"bin/llvm-objcopy.exe\",\n \"bin/wasm-ctor-eval.exe\",\n \"bin/wasm-emscripten-finalize.exe\",\n \"bin/wasm-ld.exe\",\n \"bin/wasm-metadce.exe\",\n \"bin/wasm-opt.exe\",\n \"bin/wasm-split.exe\",\n \"bin/wasm2js.exe\",\n \":emcc_common\",\n ] + glob(\n include = [\n \"emscripten/cache/sysroot/lib/**\",\n \"emscripten/node_modules/**\",\n \"emscripten/src/**\",\n ],\n ),\n)\n\nfilegroup(\n name = \"ar_files\",\n srcs = [\n \"bin/llvm-ar.exe\",\n \"emscripten/emar.py\",\n \"emscripten/emscripten-version.txt\",\n \"emscripten/src/settings.js\",\n \"emscripten/src/settings_internal.js\",\n ] + glob(\n include = [\n \"emscripten/tools/**\",\n ],\n exclude = [\n \"**/__pycache__/**\",\n ],\n ),\n)\n", - "sha256": "bd2094ca9bde5df25020a46ece7f56b622d1d22214fbd12950b01b952dd40084", - "strip_prefix": "install", - "type": "zip", - "url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/win/7033fec38817ec01909b044ea0193ddd5057255c/wasm-binaries.zip" - } - } - }, - "recordedRepoMappingEntries": [ - [ - "bazel_tools", - "rules_cc", - "rules_cc+" - ], - [ - "emsdk+", - "bazel_tools", - "bazel_tools" - ] - ] - } - }, "@@googleapis+//:extensions.bzl%switched_rules": { "general": { "bzlTransitiveDigest": "vG6fuTzXD8MMvHWZEQud0MMH7eoC4GXY0va7VrFFh04=", @@ -1322,163 +894,6 @@ ] ] } - }, - "@@rules_nodejs+//nodejs:extensions.bzl%node": { - "general": { - "bzlTransitiveDigest": "rphcryfYrOY/P3emfTskC/GY5YuHcwMl2B2ncjaM8lY=", - "usagesDigest": "u5PYlUfR7aYByqkL169Tmxkewow8vs35XHttkoUSn0U=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "nodejs_linux_amd64": { - "repoRuleId": "@@rules_nodejs+//nodejs:repositories.bzl%_nodejs_repositories", - "attributes": { - "node_download_auth": {}, - "node_repositories": {}, - "node_urls": [ - "https://nodejs.org/dist/v{version}/{filename}" - ], - "node_version": "20.18.0", - "include_headers": false, - "platform": "linux_amd64" - } - }, - "nodejs_linux_arm64": { - "repoRuleId": "@@rules_nodejs+//nodejs:repositories.bzl%_nodejs_repositories", - "attributes": { - "node_download_auth": {}, - "node_repositories": {}, - "node_urls": [ - "https://nodejs.org/dist/v{version}/{filename}" - ], - "node_version": "20.18.0", - "include_headers": false, - "platform": "linux_arm64" - } - }, - "nodejs_linux_s390x": { - "repoRuleId": "@@rules_nodejs+//nodejs:repositories.bzl%_nodejs_repositories", - "attributes": { - "node_download_auth": {}, - "node_repositories": {}, - "node_urls": [ - "https://nodejs.org/dist/v{version}/{filename}" - ], - "node_version": "20.18.0", - "include_headers": false, - "platform": "linux_s390x" - } - }, - "nodejs_linux_ppc64le": { - "repoRuleId": "@@rules_nodejs+//nodejs:repositories.bzl%_nodejs_repositories", - "attributes": { - "node_download_auth": {}, - "node_repositories": {}, - "node_urls": [ - "https://nodejs.org/dist/v{version}/{filename}" - ], - "node_version": "20.18.0", - "include_headers": false, - "platform": "linux_ppc64le" - } - }, - "nodejs_darwin_amd64": { - "repoRuleId": "@@rules_nodejs+//nodejs:repositories.bzl%_nodejs_repositories", - "attributes": { - "node_download_auth": {}, - "node_repositories": {}, - "node_urls": [ - "https://nodejs.org/dist/v{version}/{filename}" - ], - "node_version": "20.18.0", - "include_headers": false, - "platform": "darwin_amd64" - } - }, - "nodejs_darwin_arm64": { - "repoRuleId": "@@rules_nodejs+//nodejs:repositories.bzl%_nodejs_repositories", - "attributes": { - "node_download_auth": {}, - "node_repositories": {}, - "node_urls": [ - "https://nodejs.org/dist/v{version}/{filename}" - ], - "node_version": "20.18.0", - "include_headers": false, - "platform": "darwin_arm64" - } - }, - "nodejs_windows_amd64": { - "repoRuleId": "@@rules_nodejs+//nodejs:repositories.bzl%_nodejs_repositories", - "attributes": { - "node_download_auth": {}, - "node_repositories": {}, - "node_urls": [ - "https://nodejs.org/dist/v{version}/{filename}" - ], - "node_version": "20.18.0", - "include_headers": false, - "platform": "windows_amd64" - } - }, - "nodejs": { - "repoRuleId": "@@rules_nodejs+//nodejs/private:nodejs_repo_host_os_alias.bzl%nodejs_repo_host_os_alias", - "attributes": { - "user_node_repository_name": "nodejs" - } - }, - "nodejs_host": { - "repoRuleId": "@@rules_nodejs+//nodejs/private:nodejs_repo_host_os_alias.bzl%nodejs_repo_host_os_alias", - "attributes": { - "user_node_repository_name": "nodejs" - } - }, - "nodejs_toolchains": { - "repoRuleId": "@@rules_nodejs+//nodejs/private:nodejs_toolchains_repo.bzl%nodejs_toolchains_repo", - "attributes": { - "user_node_repository_name": "nodejs" - } - } - }, - "recordedRepoMappingEntries": [] - } - }, - "@@rules_python+//python/uv:uv.bzl%uv": { - "general": { - "bzlTransitiveDigest": "Xpqjnjzy6zZ90Es9Wa888ZLHhn7IsNGbph/e6qoxzw8=", - "usagesDigest": "vJ5RHUxAnV24M5swNGiAnkdxMx3Hp/iOLmNANTC5Xc8=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "uv": { - "repoRuleId": "@@rules_python+//python/uv/private:uv_toolchains_repo.bzl%uv_toolchains_repo", - "attributes": { - "toolchain_type": "'@@rules_python+//python/uv:uv_toolchain_type'", - "toolchain_names": [ - "none" - ], - "toolchain_implementations": { - "none": "'@@rules_python+//python:none'" - }, - "toolchain_compatible_with": { - "none": [ - "@platforms//:incompatible" - ] - }, - "toolchain_target_settings": {} - } - } - }, - "recordedRepoMappingEntries": [ - [ - "rules_python+", - "platforms", - "platforms" - ] - ] - } } } } diff --git a/grpc/grpcpp-c/include/protowire.h b/grpc/grpcpp-c/include/protowire.h index 1c62a7729..e53d3f41a 100644 --- a/grpc/grpcpp-c/include/protowire.h +++ b/grpc/grpcpp-c/include/protowire.h @@ -19,7 +19,6 @@ extern "C" { void pw_string_delete(pw_string_t *self); const char * pw_string_c_str(pw_string_t *self); - //// WIRE ENCODER //// typedef struct pw_encoder pw_encoder_t; @@ -52,7 +51,7 @@ extern "C" { bool pw_encoder_write_bytes(pw_encoder_t *self, int field_no, const void *data, int size); // No tag writers - bool pw_encoder_write_tag(pw_encoder_t *self, uint32_t tag); + bool pw_encoder_write_tag(pw_encoder_t *self, int field_no, int wire_type); bool pw_encoder_write_bool_no_tag(pw_encoder_t *self, bool value); bool pw_encoder_write_int32_no_tag(pw_encoder_t *self, int32_t value); bool pw_encoder_write_int64_no_tag(pw_encoder_t *self, int64_t value); @@ -114,8 +113,28 @@ extern "C" { // To read an actual bytes field, you must combine read_int32 and this function bool pw_decoder_read_raw_bytes(pw_decoder_t *self, void* buffer, int size); + /** + * Pushes the limit of the underlying pb::io::CodedStream to a certain value. + * + * This is required for reading packed fields that don't have fixed size (like int32). + * In this case, the user must push the limit by the length of the decoded LEN and read until + * the limit is reached (which indicates that the end of the repeated field is reached). + */ int pw_decoder_push_limit(pw_decoder_t *self, int limit); + /** + * Resets the limit previously pushed with pw_decoder_push_limit. + * The limit argument must be the value returned by pw_decoder_push_limit. + * + * This is typically used called after the user reached the end of a packed field and wants to + * reset the stream to the state before the read started. + */ void pw_decoder_pop_limit(pw_decoder_t *self, int limit); + /** + * Returns the number of bytes until the limit of the underlying pb::io::CodedStream is reached. + * + * This is used to know when to stop reading a packed field. It must be used in combination with + * pw_decoder_push_limit and pw_decoder_pop_limit. + */ int pw_decoder_bytes_until_limit(pw_decoder_t *self); diff --git a/grpc/grpcpp-c/src/protowire.cpp b/grpc/grpcpp-c/src/protowire.cpp index 5614951ee..50700b47d 100644 --- a/grpc/grpcpp-c/src/protowire.cpp +++ b/grpc/grpcpp-c/src/protowire.cpp @@ -27,21 +27,21 @@ namespace protowire { * The ctx pointer is used to on the K/N to reference a Kotlin managed object * from within its static callback function. * - * @param ctx the context used by the K/N side to reference Kotlin managed objects. - * @param sink the K/N callback to write data into the sink + * @param thisRef the context used by the K/N side to reference Kotlin managed objects. + * @param sink_fn the K/N callback to write data into the sink */ - SinkStream(void *ctx, bool(*sink)(void *ctx, const void *buffer, int size)) - : ctx(ctx), - sink(sink) { + SinkStream(void *thisRef, bool(*sink_fn)(void *ctx, const void *buffer, int size)) + : ctx(thisRef), + sink_fn(sink_fn) { } bool Write(const void *buffer, int size) override { - return sink(ctx, buffer, size); + return sink_fn(ctx, buffer, size); } private: void *ctx; - bool (*sink)(void *ctx, const void *buffer, int size); + bool (*sink_fn)(void *ctx, const void *buffer, int size); }; /** @@ -95,29 +95,31 @@ struct pw_string { struct pw_encoder { protowire::SinkStream sinkStream; - pb::io::CopyingOutputStreamAdaptor cosa; - pb::io::CodedOutputStream cos; + pb::io::CopyingOutputStreamAdaptor copyingOutputStreamAdaptor; + pb::io::CodedOutputStream codedOutputStream; explicit pw_encoder(protowire::SinkStream sink) : sinkStream(std::move(sink)), - cosa(&sinkStream), - cos(&cosa) { - cos.EnableAliasing(true); + copyingOutputStreamAdaptor(&sinkStream), + codedOutputStream(©ingOutputStreamAdaptor) { + codedOutputStream.EnableAliasing(true); } }; struct pw_decoder { - protowire::BufferSourceStream ss; - pb::io::CodedInputStream cis; + protowire::BufferSourceStream bufferSourceStream; + pb::io::CodedInputStream codedInputStream; explicit pw_decoder(pw_zero_copy_input_t input) - : ss(input), - cis(&ss) {} + : bufferSourceStream(input), + codedInputStream(&bufferSourceStream) {} }; extern "C" { + /// STD::STRING WRAPPER IMPLEMENTATION /// + pw_string_t *pw_string_new(const char *str) { return new pw_string_t{str }; } @@ -128,6 +130,8 @@ extern "C" { return self->str.c_str(); } + /// ENCODER IMPLEMENTATION /// + pw_encoder_t *pw_encoder_new(void* ctx, bool (* sink_fn)(void* ctx, const void* buf, int size)) { auto sink = protowire::SinkStream(ctx, sink_fn); return new pw_encoder(std::move(sink)); @@ -137,21 +141,21 @@ extern "C" { delete self; } bool pw_encoder_flush(pw_encoder_t *self) { - self->cos.Trim(); - if (!self->cosa.Flush()) { + self->codedOutputStream.Trim(); + if (!self->copyingOutputStreamAdaptor.Flush()) { return false; } - return !self->cos.HadError(); + return !self->codedOutputStream.HadError(); } // check that there was no error static bool check(pw_encoder_t *self) { - return !self->cos.HadError(); + return !self->codedOutputStream.HadError(); } #define WRITE_FIELD_FUNC( funcSuffix, wireTy, cTy) \ bool pw_encoder_write_##funcSuffix(pw_encoder_t *self, int field_no, cTy value) { \ - WireFormatLite::Write##wireTy(field_no, value, &self->cos); \ + WireFormatLite::Write##wireTy(field_no, value, &self->codedOutputStream); \ return check(self); \ } @@ -174,22 +178,22 @@ extern "C" { return pw_encoder_write_bytes(self, field_no, data, size); } bool pw_encoder_write_bytes(pw_encoder_t *self, int field_no, const void *data, int size) { - WireFormatLite::WriteTag(field_no, WireFormatLite::WIRETYPE_LENGTH_DELIMITED, &self->cos); - self->cos.WriteVarint32(size); - self->cos.WriteRawMaybeAliased(data, size); + WireFormatLite::WriteTag(field_no, WireFormatLite::WIRETYPE_LENGTH_DELIMITED, &self->codedOutputStream); + self->codedOutputStream.WriteVarint32(size); + self->codedOutputStream.WriteRawMaybeAliased(data, size); return check(self); } - bool pw_encoder_write_tag(pw_encoder_t *self, uint32_t tag) { - self->cos.WriteTag(tag); + bool pw_encoder_write_tag(pw_encoder_t *self, int field_no, int wire_type) { + WireFormatLite::WriteTag(field_no, static_cast(wire_type), &self->codedOutputStream); return check(self); } #define WRITE_FIELD_NO_TAG( funcSuffix, wireTy, cTy) \ -bool pw_encoder_write_##funcSuffix##_no_tag(pw_encoder_t *self, cTy value) { \ -WireFormatLite::Write##wireTy##NoTag(value, &self->cos); \ -return check(self); \ -} + bool pw_encoder_write_##funcSuffix##_no_tag(pw_encoder_t *self, cTy value) { \ + WireFormatLite::Write##wireTy##NoTag(value, &self->codedOutputStream); \ + return check(self); \ + } WRITE_FIELD_NO_TAG( bool, Bool, bool) WRITE_FIELD_NO_TAG( int32, Int32, int32_t) @@ -207,7 +211,7 @@ return check(self); \ WRITE_FIELD_NO_TAG( enum, Enum, int) - /// DECODER IMPLEMENATION /// + /// DECODER IMPLEMENTATION /// pw_decoder_t *pw_decoder_new(pw_zero_copy_input_t zero_copy_input) { return new pw_decoder_t(zero_copy_input); @@ -219,16 +223,16 @@ return check(self); \ void pw_decoder_close(pw_decoder_t *self) { // the deconstructor backs the stream up to the current position. - self->cis.~CodedInputStream(); + self->codedInputStream.~CodedInputStream(); } uint32_t pw_decoder_read_tag(pw_decoder_t *self) { - return self->cis.ReadTag(); + return self->codedInputStream.ReadTag(); } #define READ_VAL_FUNC( funcSuffix, wireTy, cTy) \ bool pw_decoder_read_##funcSuffix(pw_decoder_t *self, cTy *value_ref) { \ - return WireFormatLite::ReadPrimitive(&self->cis, value_ref); \ + return WireFormatLite::ReadPrimitive(&self->codedInputStream, value_ref); \ } READ_VAL_FUNC( bool, BOOL, bool) @@ -248,42 +252,42 @@ return check(self); \ bool pw_decoder_read_string(pw_decoder_t *self, pw_string_t **string_ref) { *string_ref = new pw_string_t; - return WireFormatLite::ReadString(&self->cis, &(*string_ref)->str); + return WireFormatLite::ReadString(&self->codedInputStream, &(*string_ref)->str); } bool pw_decoder_read_raw_bytes(pw_decoder_t *self, void* buffer, int size) { - return self->cis.ReadRaw(buffer, size); + return self->codedInputStream.ReadRaw(buffer, size); } int pw_decoder_push_limit(pw_decoder_t *self, int limit) { - return self->cis.PushLimit(limit); + return self->codedInputStream.PushLimit(limit); } void pw_decoder_pop_limit(pw_decoder_t *self, int limit) { - self->cis.PopLimit(limit); + self->codedInputStream.PopLimit(limit); } int pw_decoder_bytes_until_limit(pw_decoder_t *self) { - return self->cis.BytesUntilLimit(); + return self->codedInputStream.BytesUntilLimit(); } uint32_t pw_size_int32(int32_t value) { - return WireFormatLite::Int32Size(value); - } + return WireFormatLite::Int32Size(value); + } uint32_t pw_size_int64(int64_t value) { - return WireFormatLite::Int64Size(value); - } + return WireFormatLite::Int64Size(value); + } uint32_t pw_size_uint32(uint32_t value) { - return WireFormatLite::UInt32Size(value); - } + return WireFormatLite::UInt32Size(value); + } uint32_t pw_size_uint64(uint64_t value) { - return WireFormatLite::UInt64Size(value); - } + return WireFormatLite::UInt64Size(value); + } uint32_t pw_size_sint32(int32_t value) { - return WireFormatLite::SInt32Size(value); - } + return WireFormatLite::SInt32Size(value); + } uint32_t pw_size_sint64(int64_t value) { - return WireFormatLite::SInt64Size(value); - } + return WireFormatLite::SInt64Size(value); + } } From 5f80a26394ac9c9ad19c33344245da867cc52305 Mon Sep 17 00:00:00 2001 From: Alexander Sysoev Date: Mon, 28 Jul 2025 19:40:10 +0200 Subject: [PATCH 15/16] Added exclude props to grpc-ktor-server --- grpc/grpc-ktor-server/gradle.properties | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/grpc/grpc-ktor-server/gradle.properties b/grpc/grpc-ktor-server/gradle.properties index b68c20f8d..1c532ca1e 100644 --- a/grpc/grpc-ktor-server/gradle.properties +++ b/grpc/grpc-ktor-server/gradle.properties @@ -3,3 +3,18 @@ # kotlinx.rpc.exclude.wasmWasi=true +kotlinx.rpc.exclude.iosArm64=true +kotlinx.rpc.exclude.iosX64=true +kotlinx.rpc.exclude.iosSimulatorArm64=true +kotlinx.rpc.exclude.linuxArm64=true +kotlinx.rpc.exclude.linuxX64=true +kotlinx.rpc.exclude.macosX64=true +kotlinx.rpc.exclude.mingwX64=true +kotlinx.rpc.exclude.tvosArm64=true +kotlinx.rpc.exclude.tvosSimulatorArm64=true +kotlinx.rpc.exclude.tvosX64=true +kotlinx.rpc.exclude.watchosArm32=true +kotlinx.rpc.exclude.watchosArm64=true +kotlinx.rpc.exclude.watchosDeviceArm64=true +kotlinx.rpc.exclude.watchosSimulatorArm64=true +kotlinx.rpc.exclude.watchosX64=true From 3aa310e5646d6a4b1ab2ba8b051362ddfacf6620 Mon Sep 17 00:00:00 2001 From: Johannes Zottele Date: Mon, 28 Jul 2025 20:27:58 +0200 Subject: [PATCH 16/16] grpc-native: Refactor WireDecoder to avoid boxing nullable primitives Signed-off-by: Johannes Zottele --- .../kotlinx/rpc/grpc/internal/WireDecoder.kt | 72 +++---- .../rpc/grpc/internal/WireDecoder.native.kt | 192 +++++++++--------- .../rpc/grpc/internal/WireCodecTest.kt | 14 +- 3 files changed, 133 insertions(+), 145 deletions(-) diff --git a/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.kt b/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.kt index 80c3e379d..ece585248 100644 --- a/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.kt +++ b/grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.kt @@ -11,13 +11,11 @@ import kotlinx.io.Buffer * * This decoder is used by first calling [readTag], than looking up the field based on the field number in the returned, * tag and then calling the actual `read*()` method to read the value to the corresponding field. - * This means that the nullable return value does not collide with optional fields, as optional fields would not - * include a tag in the encoded message. * - * If one `read*()` method returns `null`, decoding the data failed and no further - * decoding can be done. + * [hadError] indicates an error during decoding. While calling `read*()` is safe, the returned values + * are meaningless if [hadError] returns `true`. * - * NOTE: If the value of a `read*()` method is non-null, it doesn't mean that the + * NOTE: If the [hadError] after a call to `read*()` returns `false`, it doesn't mean that the * value is correctly decoded. E.g., the following test will pass: * ```kt * val fieldNr = 1 @@ -29,43 +27,45 @@ import kotlinx.io.Buffer * * WireDecoder(buffer).use { decoder -> * decoder.readTag() - * assertNotNull(decoder.readBool()) + * decoder.readBool() + * assertFalse(decoder.hasError()) * } * ``` */ internal interface WireDecoder : AutoCloseable { + fun hadError(): Boolean fun readTag(): KTag? - fun readBool(): Boolean? - fun readInt32(): Int? - fun readInt64(): Long? - fun readUInt32(): UInt? - fun readUInt64(): ULong? - fun readSInt32(): Int? - fun readSInt64(): Long? - fun readFixed32(): UInt? - fun readFixed64(): ULong? - fun readSFixed32(): Int? - fun readSFixed64(): Long? - fun readFloat(): Float? - fun readDouble(): Double? + fun readBool(): Boolean + fun readInt32(): Int + fun readInt64(): Long + fun readUInt32(): UInt + fun readUInt64(): ULong + fun readSInt32(): Int + fun readSInt64(): Long + fun readFixed32(): UInt + fun readFixed64(): ULong + fun readSFixed32(): Int + fun readSFixed64(): Long + fun readFloat(): Float + fun readDouble(): Double - fun readEnum(): Int? - fun readString(): String? - fun readBytes(): ByteArray? - fun readPackedBool(): List? - fun readPackedInt32(): List? - fun readPackedInt64(): List? - fun readPackedSInt32(): List? - fun readPackedSInt64(): List? - fun readPackedUInt32(): List? - fun readPackedUInt64(): List? - fun readPackedFixed32(): List? - fun readPackedFixed64(): List? - fun readPackedSFixed32(): List? - fun readPackedSFixed64(): List? - fun readPackedFloat(): List? - fun readPackedDouble(): List? - fun readPackedEnum(): List? + fun readEnum(): Int + fun readString(): String + fun readBytes(): ByteArray + fun readPackedBool(): List + fun readPackedInt32(): List + fun readPackedInt64(): List + fun readPackedSInt32(): List + fun readPackedSInt64(): List + fun readPackedUInt32(): List + fun readPackedUInt64(): List + fun readPackedFixed32(): List + fun readPackedFixed64(): List + fun readPackedSFixed32(): List + fun readPackedSFixed64(): List + fun readPackedFloat(): List + fun readPackedDouble(): List + fun readPackedEnum(): List } /** diff --git a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.native.kt b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.native.kt index 05e028711..782305b1f 100644 --- a/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.native.kt +++ b/grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.native.kt @@ -19,6 +19,8 @@ private const val MAX_PACKED_BULK_SIZE: Int = 1_000_000 @OptIn(ExperimentalForeignApi::class, ExperimentalNativeApi::class) internal class WireDecoderNative(private val source: Buffer) : WireDecoder { + private var hadError = false; + // wraps the source in a class that allows to pass data from the source buffer to the C++ encoder // without copying it to an intermediate byte array. private val zeroCopyInput = StableRef.create(ZeroCopyInputSource(source)) @@ -61,149 +63,130 @@ internal class WireDecoderNative(private val source: Buffer) : WireDecoder { zeroCopyInput.dispose() } + override fun hadError(): Boolean { + return hadError; + } + override fun readTag(): KTag? { val tag = pw_decoder_read_tag(raw) - return KTag.fromOrNull(tag) + if (tag == 0u) return null.withError() + val kTag = KTag.fromOrNull(tag) + if (kTag == null) { + hadError = true + } + return kTag } - override fun readBool(): Boolean? = memScoped { + override fun readBool(): Boolean = memScoped { val value = alloc() - if (pw_decoder_read_bool(raw, value.ptr)) { - return value.value - } - return null + pw_decoder_read_bool(raw, value.ptr).checkError() + return value.value } - override fun readInt32(): Int? = memScoped { + override fun readInt32(): Int = memScoped { val value = alloc() - if (pw_decoder_read_int32(raw, value.ptr)) { - return value.value - } - return null + pw_decoder_read_int32(raw, value.ptr).checkError() + return value.value } - override fun readInt64(): Long? = memScoped { + override fun readInt64(): Long = memScoped { val value = alloc() - if (pw_decoder_read_int64(raw, value.ptr)) { - return value.value - } - return null + pw_decoder_read_int64(raw, value.ptr).checkError() + return value.value } - override fun readUInt32(): UInt? = memScoped { + override fun readUInt32(): UInt = memScoped { val value = alloc() - if (pw_decoder_read_uint32(raw, value.ptr)) { - return value.value - } - return null + pw_decoder_read_uint32(raw, value.ptr).checkError() + return value.value } - override fun readUInt64(): ULong? = memScoped { + override fun readUInt64(): ULong = memScoped { val value = alloc() - if (pw_decoder_read_uint64(raw, value.ptr)) { - return value.value - } - return null + pw_decoder_read_uint64(raw, value.ptr).checkError() + return value.value } - override fun readSInt32(): Int? = memScoped { + override fun readSInt32(): Int = memScoped { val value = alloc() - if (pw_decoder_read_sint32(raw, value.ptr)) { - return value.value - } - return null + pw_decoder_read_sint32(raw, value.ptr).checkError() + return value.value } - override fun readSInt64(): Long? = memScoped { + override fun readSInt64(): Long = memScoped { val value = alloc() - if (pw_decoder_read_sint64(raw, value.ptr)) { - return value.value - } - return null + pw_decoder_read_sint64(raw, value.ptr).checkError() + return value.value } - override fun readFixed32(): UInt? = memScoped { + override fun readFixed32(): UInt = memScoped { val value = alloc() - if (pw_decoder_read_fixed32(raw, value.ptr)) { - return value.value - } - return null + pw_decoder_read_fixed32(raw, value.ptr).checkError() + return value.value } - override fun readFixed64(): ULong? = memScoped { + override fun readFixed64(): ULong = memScoped { val value = alloc() - if (pw_decoder_read_fixed64(raw, value.ptr)) { - return value.value - } - return null + pw_decoder_read_fixed64(raw, value.ptr).checkError() + return value.value } - override fun readSFixed32(): Int? = memScoped { + override fun readSFixed32(): Int = memScoped { val value = alloc() - if (pw_decoder_read_sfixed32(raw, value.ptr)) { - return value.value - } - return null + pw_decoder_read_sfixed32(raw, value.ptr).checkError() + return value.value } - override fun readSFixed64(): Long? = memScoped { + override fun readSFixed64(): Long = memScoped { val value = alloc() - if (pw_decoder_read_sfixed64(raw, value.ptr)) { - return value.value - } - return null + pw_decoder_read_sfixed64(raw, value.ptr).checkError() + return value.value } - override fun readFloat(): Float? = memScoped { + override fun readFloat(): Float = memScoped { val value = alloc() - if (pw_decoder_read_float(raw, value.ptr)) { - return value.value - } - return null + pw_decoder_read_float(raw, value.ptr).checkError() + return value.value } - override fun readDouble(): Double? = memScoped { + override fun readDouble(): Double = memScoped { val value = alloc() - if (pw_decoder_read_double(raw, value.ptr)) { - return value.value - } - return null + pw_decoder_read_double(raw, value.ptr).checkError() + return value.value } - override fun readEnum(): Int? = memScoped { + override fun readEnum(): Int = memScoped { val value = alloc() - if (pw_decoder_read_enum(raw, value.ptr)) { - return value.value - } - return null + pw_decoder_read_enum(raw, value.ptr).checkError() + return value.value } // TODO: Is it possible to avoid copying the c_str, by directly allocating a K/N String (as in readBytes)? KRPC-187 - override fun readString(): String? = memScoped { + override fun readString(): String = memScoped { val str = alloc>() - val ok = pw_decoder_read_string(raw, str.ptr) + pw_decoder_read_string(raw, str.ptr).checkError() try { - if (!ok) return null - return pw_string_c_str(str.value)?.toKString() + if (hadError) return "" + return pw_string_c_str(str.value)?.toKString() ?: "".also { hadError = true } } finally { pw_string_delete(str.value) } } // TODO: Should readBytes return a buffer, to prevent allocation of large contiguous memory blocks ? KRPC-182 - override fun readBytes(): ByteArray? { - val length = readInt32() ?: return null - if (length < 0) return null + override fun readBytes(): ByteArray { + val length = readInt32() + if (hadError) return ByteArray(0) + if (length < 0) return ByteArray(0).withError() // check if the remaining buffer size is less than the set length, // we can early abort, without allocating unnecessary memory - if (source.size < length) return null - if (length == 0) return ByteArray(0) + if (source.size < length) return ByteArray(0).withError() + if (length == 0) return ByteArray(0) // actually an empty array (no error) val bytes = ByteArray(length) - var ok = true bytes.usePinned { - ok = pw_decoder_read_raw_bytes(raw, it.addressOf(0), length) + pw_decoder_read_raw_bytes(raw, it.addressOf(0), length).checkError() } - if (!ok) return null + if (hadError) return ByteArray(0) return bytes } @@ -259,19 +242,21 @@ internal class WireDecoderNative(private val source: Buffer) : WireDecoder { ) private inline fun readPackedVarInternal( - crossinline readFn: () -> T? - ): List? { - val byteLen = readInt32() ?: return null - if (byteLen < 0) return null - if (source.size < byteLen) return null - if (byteLen == 0) return emptyList() + crossinline readFn: () -> T + ): List { + val byteLen = readInt32() + if (hadError) return emptyList() + if (byteLen < 0) return emptyList().withError() + if (source.size < byteLen) return emptyList().withError() + if (byteLen == 0) return emptyList() // actually an empty list (no error) val limit = pw_decoder_push_limit(raw, byteLen) val result = mutableListOf() while (pw_decoder_bytes_until_limit(raw) > 0) { - val elem = readFn() ?: return null + val elem = readFn() + if (hadError) break result.add(elem) } @@ -293,13 +278,14 @@ internal class WireDecoderNative(private val source: Buffer) : WireDecoder { crossinline createArray: (Int) -> R, crossinline getAddress: Pinned.(Int) -> COpaquePointer, crossinline asList: (R) -> List - ): List? { + ): List { // fetch the size of the packed repeated field - var byteLen = readInt32() ?: return null - if (byteLen < 0) return null - if (source.size < byteLen) return null - if (byteLen % sizeBytes != 0) return null - if (byteLen == 0) return emptyList() + var byteLen = readInt32() + if (hadError) return emptyList() + if (byteLen < 0) return emptyList().withError() + if (source.size < byteLen) return emptyList().withError() + if (byteLen % sizeBytes != 0) return emptyList().withError() + if (byteLen == 0) return emptyList() // actually an empty list (no error) // allocate the buffer array (has at most MAX_PACKED_BULK_SIZE bytes) val bufByteLen = minOf(byteLen, MAX_PACKED_BULK_SIZE) @@ -311,7 +297,7 @@ internal class WireDecoderNative(private val source: Buffer) : WireDecoder { if (byteLen == bufByteLen) { // the whole packed field fits into the buffer -> copy into buffer and returns it as a list. - pw_decoder_read_raw_bytes(raw, bufAddr, byteLen) + pw_decoder_read_raw_bytes(raw, bufAddr, byteLen).checkError() return asList(buffer) } else { // the packed field is too large for the buffer, so we load it into a persistent list @@ -320,7 +306,8 @@ internal class WireDecoderNative(private val source: Buffer) : WireDecoder { while (byteLen > 0) { // copy data into the buffer. val copySize = min(bufByteLen, byteLen) - pw_decoder_read_raw_bytes(raw, bufAddr, copySize) + pw_decoder_read_raw_bytes(raw, bufAddr, copySize).checkError() + if (hadError) return emptyList() // add buffer to the chunked list chunkedList = if (copySize == bufByteLen) { @@ -336,6 +323,15 @@ internal class WireDecoderNative(private val source: Buffer) : WireDecoder { } } } + + private fun Boolean.checkError() { + hadError = !this || hadError; + } + + private fun T.withError(): T { + hadError = true + return this + } } internal actual fun WireDecoder(source: Buffer): WireDecoder = WireDecoderNative(source) \ No newline at end of file diff --git a/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/WireCodecTest.kt b/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/WireCodecTest.kt index 19bdf1333..d93132fc1 100644 --- a/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/WireCodecTest.kt +++ b/grpc/grpc-core/src/nativeTest/kotlin/kotlinx/rpc/grpc/internal/WireCodecTest.kt @@ -25,6 +25,7 @@ class WireCodecTest { val decoder = WireDecoder(buffer) val tag = decoder.readTag() + assertFalse(decoder.hadError()) assertNotNull(tag) assertEquals(WireType.VARINT, tag.wireType) assertEquals(fieldNr, tag.fieldNr) @@ -342,17 +343,8 @@ class WireCodecTest { val buffer = Buffer() val decoder = WireDecoder(buffer) - assertNull(decoder.readTag()) - assertNull(decoder.readBool()) - assertNull(decoder.readInt32()) - assertNull(decoder.readInt64()) - assertNull(decoder.readSInt32()) - assertNull(decoder.readSInt64()) - assertNull(decoder.readUInt32()) - assertNull(decoder.readUInt64()) - assertNull(decoder.readString()) - assertNull(decoder.readEnum()) - + decoder.readTag() + assertTrue(decoder.hadError()) } @Test