Cbor: check if inline value classes is marked as @ByteString (#2466)

Fixes #2187

Author: the-eater

formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt

@@ -70,6 +70,8 @@ internal open class CborWriter(private val cbor: Cbor, protected val encoder: Cb
         if (encodeByteArrayAsByteString && serializer.descriptor == ByteArraySerializer().descriptor) {
             encoder.encodeByteString(value as ByteArray)
         } else {
+            encodeByteArrayAsByteString = encodeByteArrayAsByteString || serializer.descriptor.isInlineByteString()
+
             super.encodeSerializableValue(serializer, value)
         }
     }
@@ -278,6 +280,7 @@ internal open class CborReader(private val cbor: Cbor, protected val decoder: Cb
             @Suppress("UNCHECKED_CAST")
             decoder.nextByteString() as T
         } else {
+            decodeByteArrayAsByteString = decodeByteArrayAsByteString || deserializer.descriptor.isInlineByteString()
             super.decodeSerializableValue(deserializer)
         }
     }
@@ -636,6 +639,11 @@ private fun SerialDescriptor.isByteString(index: Int): Boolean {
     return getElementAnnotations(index).find { it is ByteString } != null
 }
 
+private fun SerialDescriptor.isInlineByteString(): Boolean {
+    // inline item classes should only have 1 item
+    return isInline && isByteString(0)
+}
+
 
 private val normalizeBaseBits = SINGLE_PRECISION_NORMALIZE_BASE.toBits()
 

formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborReaderTest.kt

@@ -633,6 +633,34 @@ class CborReaderTest {
         )
     }
 
+    @Test
+    fun testReadValueClassWithByteString() {
+        assertContentEquals(
+            expected = byteArrayOf(0x11, 0x22, 0x33),
+            actual = Cbor.decodeFromHexString<ValueClassWithByteString>("43112233").x
+        )
+    }
+
+    @Test
+    fun testReadValueClassCustomByteString() {
+        assertEquals(
+            expected = ValueClassWithCustomByteString(CustomByteString(0x11, 0x22, 0x33)),
+            actual = Cbor.decodeFromHexString("43112233")
+        )
+    }
+
+    @Test
+    fun testReadValueClassWithUnlabeledByteString() {
+        assertContentEquals(
+            expected = byteArrayOf(
+                0x11,
+                0x22,
+                0x33
+            ),
+            actual = Cbor.decodeFromHexString<ValueClassWithUnlabeledByteString>("43112233").x.x
+        )
+    }
+
     @Test
     fun testIgnoresTagsOnStrings() {
         /*

formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborWriterTest.kt

@@ -122,4 +122,28 @@ class CbrWriterTest {
                 actual = Cbor.encodeToHexString(TypeWithNullableCustomByteString(null))
         )
     }
+
+    @Test
+    fun testWriteValueClassWithByteString() {
+        assertEquals(
+            expected = "43112233",
+            actual = Cbor.encodeToHexString(ValueClassWithByteString(byteArrayOf(0x11, 0x22, 0x33)))
+        )
+    }
+
+    @Test
+    fun testWriteValueClassCustomByteString() {
+        assertEquals(
+            expected = "43112233",
+            actual = Cbor.encodeToHexString(ValueClassWithCustomByteString(CustomByteString(0x11, 0x22, 0x33)))
+        )
+    }
+
+    @Test
+    fun testWriteValueClassWithUnlabeledByteString() {
+        assertEquals(
+            expected = "43112233",
+            actual = Cbor.encodeToHexString(ValueClassWithUnlabeledByteString(ValueClassWithUnlabeledByteString.Inner(byteArrayOf(0x11, 0x22, 0x33))))
+        )
+    }
 }

formats/cbor/commonTest/src/kotlinx/serialization/cbor/SampleClasses.kt

@@ -8,6 +8,7 @@ import kotlinx.serialization.*
 import kotlinx.serialization.builtins.*
 import kotlinx.serialization.descriptors.*
 import kotlinx.serialization.encoding.*
+import kotlin.jvm.*
 
 @Serializable
 data class Simple(val a: String)
@@ -110,4 +111,20 @@ class CustomByteStringSerializer : KSerializer<CustomByteString> {
 data class TypeWithCustomByteString(@ByteString val x: CustomByteString)
 
 @Serializable
-data class TypeWithNullableCustomByteString(@ByteString val x: CustomByteString?)
+data class TypeWithNullableCustomByteString(@ByteString val x: CustomByteString?)
+
+@JvmInline
+@Serializable
+value class ValueClassWithByteString(@ByteString val x: ByteArray)
+
+@JvmInline
+@Serializable
+value class ValueClassWithCustomByteString(@ByteString val x: CustomByteString)
+
+@JvmInline
+@Serializable
+value class ValueClassWithUnlabeledByteString(@ByteString val x: Inner) {
+    @JvmInline
+    @Serializable
+    value class Inner(val x: ByteArray)
+}