Skip to content

Commit 202c44d

Browse files
Jozott00Jozott00
committed
grpc-native: Add variable length packed field encoding
Signed-off-by: Johannes Zottele <[email protected]>
1 parent 1fa0527 commit 202c44d

File tree

12 files changed

+339
-19
lines changed

12 files changed

+339
-19
lines changed

‎grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.kt‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,12 +46,20 @@ internal interface WireDecoder : AutoCloseable {
4646
fun readEnum(): Int?
4747
fun readString(): String?
4848
fun readBytes(): ByteArray?
49+
fun readPackedBool(): List<Boolean>?
50+
fun readPackedInt32(): List<Int>?
51+
fun readPackedInt64(): List<Long>?
52+
fun readPackedSInt32(): List<Int>?
53+
fun readPackedSInt64(): List<Long>?
54+
fun readPackedUInt32(): List<UInt>?
55+
fun readPackedUInt64(): List<ULong>?
4956
fun readPackedFixed32(): List<UInt>?
5057
fun readPackedFixed64(): List<ULong>?
5158
fun readPackedSFixed32(): List<Int>?
5259
fun readPackedSFixed64(): List<Long>?
5360
fun readPackedFloat(): List<Float>?
5461
fun readPackedDouble(): List<Double>?
62+
fun readPackedEnum(): List<Int>?
5563
}
5664

5765
/**

‎grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.kt‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,12 +33,21 @@ internal interface WireEncoder {
3333
fun writeEnum(fieldNr: Int, value: Int): Boolean
3434
fun writeBytes(fieldNr: Int, value: ByteArray): Boolean
3535
fun writeString(fieldNr: Int, value: String): Boolean
36+
fun writePackedBool(fieldNr: Int, value: List<Boolean>, fieldSize: Int): Boolean
37+
fun writePackedInt32(fieldNr: Int, value: List<Int>, fieldSize: Int): Boolean
38+
fun writePackedInt64(fieldNr: Int, value: List<Long>, fieldSize: Int): Boolean
39+
fun writePackedUInt32(fieldNr: Int, value: List<UInt>, fieldSize: Int): Boolean
40+
fun writePackedUInt64(fieldNr: Int, value: List<ULong>, fieldSize: Int): Boolean
41+
fun writePackedSInt32(fieldNr: Int, value: List<Int>, fieldSize: Int): Boolean
42+
fun writePackedSInt64(fieldNr: Int, value: List<Long>, fieldSize: Int): Boolean
3643
fun writePackedFixed32(fieldNr: Int, value: List<UInt>): Boolean
3744
fun writePackedFixed64(fieldNr: Int, value: List<ULong>): Boolean
3845
fun writePackedSFixed32(fieldNr: Int, value: List<Int>): Boolean
3946
fun writePackedSFixed64(fieldNr: Int, value: List<Long>): Boolean
4047
fun writePackedFloat(fieldNr: Int, value: List<Float>): Boolean
4148
fun writePackedDouble(fieldNr: Int, value: List<Double>): Boolean
49+
fun writePackedEnum(fieldNr: Int, value: List<Int>, fieldSize: Int) =
50+
writePackedInt32(fieldNr, value, fieldSize)
4251
}
4352

4453

‎grpc/grpc-core/src/commonMain/kotlin/kotlinx/rpc/grpc/internal/WireSize.kt‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
/*
2+
* Copyright 2023-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
3+
*/
4+
5+
package kotlinx.rpc.grpc.internal
6+
7+
internal object WireSize
8+
9+
internal expect fun WireSize.int32(value: Int): UInt
10+
internal expect fun WireSize.int64(value: Long): UInt
11+
internal expect fun WireSize.uInt32(value: UInt): UInt
12+
internal expect fun WireSize.uInt64(value: ULong): UInt
13+
internal expect fun WireSize.sInt32(value: Int): UInt
14+
internal expect fun WireSize.sInt64(value: Long): UInt
15+
16+
internal fun WireSize.bool(value: Boolean) = int32(if (value) 1 else 0)
17+
internal fun WireSize.enum(value: Int) = int32(value)
18+
internal fun WireSize.packedInt32(value: List<Int>) = value.sumOf { int32(it) }
19+
internal fun WireSize.packedInt64(value: List<Long>) = value.sumOf { int64(it) }
20+
internal fun WireSize.packedUInt32(value: List<UInt>) = value.sumOf { uInt32(it) }
21+
internal fun WireSize.packedUInt64(value: List<ULong>) = value.sumOf { uInt64(it) }
22+
internal fun WireSize.packedSInt32(value: List<Int>) = value.sumOf { sInt32(it) }
23+
internal fun WireSize.packedSInt64(value: List<Long>) = value.sumOf { sInt64(it) }
24+
internal fun WireSize.packedEnum(value: List<Int>) = value.sumOf { enum(it) }

‎grpc/grpc-core/src/jsMain/kotlin/kotlinx/rpc/grpc/internal/WireSize.js.kt‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
/*
2+
* Copyright 2023-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
3+
*/
4+
5+
package kotlinx.rpc.grpc.internal
6+
7+
internal actual fun WireSize.int32(value: Int): UInt {
8+
TODO("Not yet implemented")
9+
}
10+
11+
internal actual fun WireSize.int64(value: Long): UInt {
12+
TODO("Not yet implemented")
13+
}
14+
15+
internal actual fun WireSize.uInt32(value: UInt): UInt {
16+
TODO("Not yet implemented")
17+
}
18+
19+
internal actual fun WireSize.uInt64(value: ULong): UInt {
20+
TODO("Not yet implemented")
21+
}
22+
23+
internal actual fun WireSize.sInt32(value: Int): UInt {
24+
TODO("Not yet implemented")
25+
}
26+
27+
internal actual fun WireSize.sInt64(value: Long): UInt {
28+
TODO("Not yet implemented")
29+
}

‎grpc/grpc-core/src/jvmMain/kotlin/kotlinx/rpc/grpc/internal/WireSize.jvm.kt‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
/*
2+
* Copyright 2023-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
3+
*/
4+
5+
package kotlinx.rpc.grpc.internal
6+
7+
internal actual fun WireSize.int32(value: Int): UInt {
8+
TODO("Not yet implemented")
9+
}
10+
11+
internal actual fun WireSize.int64(value: Long): UInt {
12+
TODO("Not yet implemented")
13+
}
14+
15+
internal actual fun WireSize.uInt32(value: UInt): UInt {
16+
TODO("Not yet implemented")
17+
}
18+
19+
internal actual fun WireSize.uInt64(value: ULong): UInt {
20+
TODO("Not yet implemented")
21+
}
22+
23+
internal actual fun WireSize.sInt32(value: Int): UInt {
24+
TODO("Not yet implemented")
25+
}
26+
27+
internal actual fun WireSize.sInt64(value: Long): UInt {
28+
TODO("Not yet implemented")
29+
}

‎grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireDecoder.native.kt‎

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ internal class WireDecoderNative(private val source: Buffer) : WireDecoder {
4545
pw_decoder_new(zeroCopyCInput)
4646
?: error("Failed to create proto wire decoder")
4747
}
48-
48+
4949
val rawCleaner = createCleaner(raw) {
5050
pw_decoder_delete(it)
5151
}
@@ -206,6 +206,15 @@ internal class WireDecoderNative(private val source: Buffer) : WireDecoder {
206206
return bytes
207207
}
208208

209+
override fun readPackedBool() = readPackedVarInternal(this::readBool)
210+
override fun readPackedInt32() = readPackedVarInternal(this::readInt32)
211+
override fun readPackedInt64() = readPackedVarInternal(this::readInt64)
212+
override fun readPackedUInt32() = readPackedVarInternal(this::readUInt32)
213+
override fun readPackedUInt64() = readPackedVarInternal(this::readUInt64)
214+
override fun readPackedSInt32() = readPackedVarInternal(this::readSInt32)
215+
override fun readPackedSInt64() = readPackedVarInternal(this::readSInt64)
216+
override fun readPackedEnum() = readPackedVarInternal(this::readEnum)
217+
209218
override fun readPackedFixed32() = readPackedFixedInternal(
210219
UInt.SIZE_BYTES,
211220
::UIntArray,
@@ -248,6 +257,27 @@ internal class WireDecoderNative(private val source: Buffer) : WireDecoder {
248257
DoubleArray::asList,
249258
)
250259

260+
private inline fun <T : Any> readPackedVarInternal(
261+
crossinline readFn: () -> T?
262+
): List<T>? {
263+
val byteLen = readInt32() ?: return null
264+
if (byteLen < 0) return null
265+
if (source.size < byteLen) return null
266+
if (byteLen == 0) return null
267+
268+
val limit = pw_decoder_push_limit(raw, byteLen)
269+
270+
val result = mutableListOf<T>()
271+
272+
while (pw_decoder_bytes_until_limit(raw) > 0) {
273+
val elem = readFn() ?: return null
274+
result.add(elem)
275+
}
276+
277+
pw_decoder_pop_limit(raw, limit)
278+
return result
279+
}
280+
251281
/*
252282
* Based on the length of the packed repeated field, one of two list strategies is chosen.
253283
* If the length is less or equal a specific threshold (MAX_PACKED_BULK_SIZE),

‎grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireEncoder.native.kt‎

Lines changed: 30 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,6 @@ import kotlin.experimental.ExperimentalNativeApi
1111
import kotlin.native.ref.createCleaner
1212

1313

14-
// TODO: Evaluate if we should implement a ZeroCopyOutputSink (similar to the ZeroCopyInputSource)
15-
// to reduce the number of copies during encoding.
1614
@OptIn(ExperimentalForeignApi::class, ExperimentalNativeApi::class)
1715
internal class WireEncoderNative(private val sink: Sink) : WireEncoder {
1816
/**
@@ -125,39 +123,61 @@ internal class WireEncoderNative(private val sink: Sink) : WireEncoder {
125123
}
126124
}
127125

126+
override fun writePackedBool(fieldNr: Int, value: List<Boolean>, fieldSize: Int) =
127+
writePackedInternal(fieldNr, value, fieldSize, ::pw_encoder_write_bool_no_tag)
128+
129+
override fun writePackedInt32(fieldNr: Int, value: List<Int>, fieldSize: Int) =
130+
writePackedInternal(fieldNr, value, fieldSize, ::pw_encoder_write_int32_no_tag)
131+
132+
override fun writePackedInt64(fieldNr: Int, value: List<Long>, fieldSize: Int) =
133+
writePackedInternal(fieldNr, value, fieldSize, ::pw_encoder_write_int64_no_tag)
134+
135+
override fun writePackedUInt32(fieldNr: Int, value: List<UInt>, fieldSize: Int) =
136+
writePackedInternal(fieldNr, value, fieldSize, ::pw_encoder_write_uint32_no_tag)
137+
138+
override fun writePackedUInt64(fieldNr: Int, value: List<ULong>, fieldSize: Int) =
139+
writePackedInternal(fieldNr, value, fieldSize, ::pw_encoder_write_uint64_no_tag)
140+
141+
override fun writePackedSInt32(fieldNr: Int, value: List<Int>, fieldSize: Int) =
142+
writePackedInternal(fieldNr, value, fieldSize, ::pw_encoder_write_sint32_no_tag)
143+
144+
override fun writePackedSInt64(fieldNr: Int, value: List<Long>, fieldSize: Int) =
145+
writePackedInternal(fieldNr, value, fieldSize, ::pw_encoder_write_sint64_no_tag)
146+
128147
override fun writePackedFixed32(fieldNr: Int, value: List<UInt>) =
129-
writePackedInternal(fieldNr, value, UInt.SIZE_BYTES, ::pw_encoder_write_fixed32_no_tag)
148+
writePackedInternal(fieldNr, value, value.size * UInt.SIZE_BYTES, ::pw_encoder_write_fixed32_no_tag)
130149

131150
override fun writePackedFixed64(fieldNr: Int, value: List<ULong>) =
132-
writePackedInternal(fieldNr, value, ULong.SIZE_BYTES, ::pw_encoder_write_fixed64_no_tag)
151+
writePackedInternal(fieldNr, value, value.size * ULong.SIZE_BYTES, ::pw_encoder_write_fixed64_no_tag)
133152

134153
override fun writePackedSFixed32(fieldNr: Int, value: List<Int>) =
135-
writePackedInternal(fieldNr, value, Int.SIZE_BYTES, ::pw_encoder_write_sfixed32_no_tag)
154+
writePackedInternal(fieldNr, value, value.size * Int.SIZE_BYTES, ::pw_encoder_write_sfixed32_no_tag)
136155

137156
override fun writePackedSFixed64(fieldNr: Int, value: List<Long>) =
138-
writePackedInternal(fieldNr, value, Long.SIZE_BYTES, ::pw_encoder_write_sfixed64_no_tag)
157+
writePackedInternal(fieldNr, value, value.size * Long.SIZE_BYTES, ::pw_encoder_write_sfixed64_no_tag)
139158

140159
override fun writePackedFloat(fieldNr: Int, value: List<Float>) =
141-
writePackedInternal(fieldNr, value, Float.SIZE_BYTES, ::pw_encoder_write_float_no_tag)
160+
writePackedInternal(fieldNr, value, value.size * Float.SIZE_BYTES, ::pw_encoder_write_float_no_tag)
142161

143162
override fun writePackedDouble(fieldNr: Int, value: List<Double>) =
144-
writePackedInternal(fieldNr, value, Double.SIZE_BYTES, ::pw_encoder_write_double_no_tag)
163+
writePackedInternal(fieldNr, value, value.size * Double.SIZE_BYTES, ::pw_encoder_write_double_no_tag)
145164
}
146165

147166
internal actual fun WireEncoder(sink: Sink): WireEncoder = WireEncoderNative(sink)
148167

149168

169+
// the current implementation is slow, as it iterates through the list, to write each element individually,
170+
// which can be speed up in case of fixed sized types, that are not compressed. KRPC-183
150171
@OptIn(ExperimentalForeignApi::class)
151172
private inline fun <T> WireEncoderNative.writePackedInternal(
152173
fieldNr: Int,
153174
value: List<T>,
154-
byteSize: Int,
175+
fieldSize: Int,
155176
crossinline writer: (CValuesRef<pw_encoder_t>?, T) -> Boolean
156177
): Boolean {
157178
val ktag = KTag(fieldNr, WireType.LENGTH_DELIMITED).toRawKTag()
158179
pw_encoder_write_tag(raw, ktag)
159180
// write the field size of the packed field
160-
val fieldSize = value.size * byteSize;
161181
pw_encoder_write_int32_no_tag(raw, fieldSize)
162182
for (v in value) {
163183
if (!writer(raw, v)) {

‎grpc/grpc-core/src/nativeMain/kotlin/kotlinx/rpc/grpc/internal/WireSize.native.kt‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
/*
2+
* Copyright 2023-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
3+
*/
4+
5+
@file:OptIn(ExperimentalForeignApi::class)
6+
7+
package kotlinx.rpc.grpc.internal
8+
9+
import kotlinx.cinterop.ExperimentalForeignApi
10+
import libprotowire.*
11+
12+
internal actual fun WireSize.int32(value: Int) = pw_size_int32(value)
13+
internal actual fun WireSize.int64(value: Long) = pw_size_int64(value)
14+
internal actual fun WireSize.uInt32(value: UInt) = pw_size_uint32(value)
15+
internal actual fun WireSize.uInt64(value: ULong) = pw_size_uint64(value)
16+
internal actual fun WireSize.sInt32(value: Int) = pw_size_sint32(value)
17+
internal actual fun WireSize.sInt64(value: Long) = pw_size_sint64(value)
18+

0 commit comments

Comments
 (0)