feat(rt): replace GSON based JSON serde with KMP compatible impl (#477)

Co-authored-by: Aaron Todd <[email protected]>
Co-authored-by: Kyle Thomson <[email protected]>

Author: aajtodd

runtime/io/common/src/aws/smithy/kotlin/runtime/io/SdkByteReadChannel.kt

@@ -4,6 +4,7 @@
  */
 package aws.smithy.kotlin.runtime.io
 
+import aws.smithy.kotlin.runtime.util.text.byteCountUtf8
 import io.ktor.utils.io.*
 import io.ktor.utils.io.core.*
 
@@ -160,3 +161,33 @@ internal suspend fun SdkByteReadChannel.readAvailableFallback(dest: SdkByteBuffe
     dest.writeFully(tmp)
     return tmp.size.toLong()
 }
+
+/**
+ * Reads a UTF-8 code point from the channel. Returns `null` if closed
+ */
+suspend fun SdkByteReadChannel.readUtf8CodePoint(): Int? {
+    awaitContent()
+    if (availableForRead == 0 && isClosedForRead) return null
+
+    val firstByte = readByte()
+    val cnt = byteCountUtf8(firstByte)
+    var code = when (cnt) {
+        1 -> firstByte.toInt()
+        2 -> firstByte.toInt() and 0x1f
+        3 -> firstByte.toInt() and 0x0f
+        4 -> firstByte.toInt() and 0x07
+        else -> throw IllegalStateException("Invalid UTF-8 start sequence: $firstByte")
+    }
+
+    for (i in 1 until cnt) {
+        awaitContent()
+        if (availableForRead == 0 && isClosedForRead) throw IllegalStateException("unexpected EOF: expected ${cnt - i} bytes")
+        val byte = readByte()
+        val bint = byte.toInt()
+        if (bint and 0xc0 != 0x80) throw IllegalStateException("invalid UTF-8 successor byte: $byte")
+
+        code = (code shl 6) or (bint and 0x3f)
+    }
+
+    return code
+}

runtime/io/common/test/aws/smithy/kotlin/runtime/io/SdkByteChannelOpsTest.kt

@@ -131,4 +131,36 @@ class SdkByteChannelOpsTest {
         yield()
         assertFalse(awaitingContent)
     }
+
+    @Test
+    fun testReadUtf8Chars() = runSuspendTest {
+        val chan = SdkByteReadChannel("hello".encodeToByteArray())
+        assertEquals('h', chan.readUtf8CodePoint()?.toChar())
+        assertEquals('e', chan.readUtf8CodePoint()?.toChar())
+        assertEquals('l', chan.readUtf8CodePoint()?.toChar())
+        assertEquals('l', chan.readUtf8CodePoint()?.toChar())
+        assertEquals('o', chan.readUtf8CodePoint()?.toChar())
+        assertNull(chan.readUtf8CodePoint())
+    }
+
+    @Test
+    fun testReadMultibyteUtf8Chars(): Unit = runSuspendTest {
+        // https://www.fileformat.info/info/unicode/char/1d122/index.htm
+        // $ - 1 byte, cent sign - 2bytes, euro sign - 3 bytes, musical clef - 4 points (surrogate pair)
+        val content = "$¢€\uD834\uDD22"
+        val chan = SdkByteReadChannel(content.encodeToByteArray())
+
+        val expected = listOf(
+            36, // $
+            162, // ¢
+            8364, // €
+            119074 // musical F clef
+        )
+
+        expected.forEachIndexed { i, exp ->
+            val code = chan.readUtf8CodePoint()
+            assertEquals(exp, code, "[i=$i] expected $exp, found $code ")
+        }
+        assertNull(chan.readUtf8CodePoint())
+    }
 }

runtime/serde/build.gradle.kts

@@ -19,7 +19,7 @@ kotlin {
     }
 }
 
-subprojects {
+allprojects {
     kotlin {
         sourceSets {
             commonTest {

runtime/serde/serde-json/TESTING.md

@@ -0,0 +1,178 @@
+How to run JSONTestSuite against serde-json deserialize
+========================================================
+
+When making changes to the lexer it is a good idea to run the
+changes against the [JSONTestSuite](https://github.com/nst/JSONTestSuite) and manually examine the test results.
+
+### How to setup the JSONTestSuite
+
+1. Clone the [JSONTestSuite](https://github.com/nst/JSONTestSuite) repository.
+2. In `JSONTestSuite/parsers`, create a new Gradle JVM application project named `test_smithy_kotlin`.
+3. Add the following `build.gradle.kts` file
+
+```kotlin
+plugins {
+    kotlin("jvm") version "1.5.30"
+    application
+    id("com.github.johnrengelman.shadow") version "7.0.0"
+}
+
+application {
+    mainClass.set("aws.smithy.kotlin.jsontest.MainKt")
+}
+
+allprojects {
+    repositories {
+        mavenLocal()
+        mavenCentral()
+    }
+}
+
+
+// NOTE: set to whatever locally published version you are working on
+val smithyKotlinVersion: String = "0.4.1-kmp-json"
+dependencies {
+   implementation(kotlin("stdlib"))
+   implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.5.0")
+   implementation("aws.smithy.kotlin:serde-json:$smithyKotlinVersion")
+   implementation("aws.smithy.kotlin:utils:$smithyKotlinVersion")
+}
+
+tasks.jar {
+    manifest {
+        attributes["Main-Class"] = "aws.smithy.kotlin.jsontest.MainKt"
+    }
+}
+```
+
+4. Add the following code to `src/main/kotlin/Main.kt` with:
+
+```kotlin
+package aws.smithy.kotlin.jsontest
+
+import kotlinx.coroutines.runBlocking
+import kotlin.system.exitProcess
+import java.io.IOException
+import java.nio.file.Files
+import java.nio.file.Paths
+import aws.smithy.kotlin.runtime.serde.json.JsonToken
+import aws.smithy.kotlin.runtime.serde.json.jsonStreamReader
+import aws.smithy.kotlin.runtime.util.InternalApi
+
+
+@OptIn(InternalApi::class)
+suspend fun isValidJson(bytes: ByteArray):Boolean {
+    val lexer = jsonStreamReader(bytes)
+    println(lexer::class.qualifiedName)
+    return try {
+        val tokens = mutableListOf<JsonToken>()
+        do {
+            val token = lexer.nextToken()
+            tokens.add(token)
+        }while(token != JsonToken.EndDocument)
+
+        // The test suite includes incomplete objects and arrays (e.g. "[null,")
+        // These are completely valid for this parser since it's just a tokenizer
+        // and doesn't attempt to make semantic meaning from the input.
+        // We'll just pretend to fail to satisfy the test suite
+        val pruned = if (tokens.last() == JsonToken.EndDocument) tokens.dropLast(1) else tokens
+        if (pruned.first() == JsonToken.BeginArray && pruned.last() != JsonToken.EndArray) {
+            return false
+        }
+        if (pruned.first() == JsonToken.BeginObject && pruned.last() != JsonToken.EndObject) {
+            return false
+        }
+
+        tokens.isNotEmpty()
+    }catch(ex: Exception) {
+        println(ex)
+        false
+    }
+}
+
+fun main(args: Array<String>): Unit = runBlocking {
+    if(args.isEmpty()) {
+        println("Usage: java TestJSONParsing file.json")
+        exitProcess(2)
+    }
+
+    try {
+        val data = Files.readAllBytes(Paths.get(args[0]))
+        if(isValidJson(data)) {
+            println("valid");
+            exitProcess(0);
+        }
+        println("invalid");
+        exitProcess(1);
+    } catch (ex: IOException) {
+        println(ex)
+        println("not found");
+        exitProcess(2);
+    }
+}
+```
+
+5. Compile this program with `./gradlew build`. 
+   NOTE: Be sure to publish all of `smithy-kotlin` "runtime" to maven local. It is helpful to just choose a unique version
+   to be sure that everything is wired up correctly.
+6. Modify `JSONTestSuite/run_tests.py` so that the `programs` dictionary only contains this one entry:
+
+```
+programs = {
+   "SmithyKotlin":
+       {
+           "url":"",
+           "commands":["java" , "-jar", os.path.join(PARSERS_DIR, "test_smithy_kotlin/build/libs/test_smithy_kotlin-all.jar")]
+       }
+}
+```
+
+7. Run `run_tests.py` and examine the output with a web browser by opening `JSONTestSuite/results/parsing.html`.
+
+### Examining the results
+
+When looking at `JSONTestSuite/results/parsing.html`, there is a matrix of test cases against their
+results with a legend at the top.
+
+Any test result marked with blue or light blue is for a test case where correct behavior isn't specified,
+so use your best judgement to decide if it should have succeeded or failed.
+
+The other colors are bad and should be carefully examined. At time of writing, the following test cases
+succeed when they should fail, and we intentionally left it that way since we're not currently concerned
+about being more lenient in the number parsing:
+
+```
+
+n_number_-01.json                               [-01]
+n_number_-2..json                               [-2.]
+n_number_.2e-3.json                             [.2e-3]
+n_number_0.3e+.json                             [0.3e+]
+n_number_0.3e.json                              [0.3e]
+n_number_0.e1.json                              [0.e1]
+n_number_0_capital_E+.json                      [0E+]
+n_number_0_capital_E.json                       [0E]
+n_number_0e+.json                               [0e+]
+n_number_0e.json                                [0e]
+n_number_1.0e+.json                             [1.0e+]
+n_number_1.0e-.json                             [1.0e-]
+n_number_1.0e.json                              [1.0e]
+n_number_2.e+3.json                             [2.e+3]
+n_number_2.e-3.json                             [2.e-3]
+n_number_2.e3.json                              [2.e3]
+n_number_9.e+.json                              [9.e+]
+n_number_neg_int_starting_with_zero.json        [-012]
+n_number_neg_real_without_int_part.json         [-.123]
+n_number_real_without_fractional_part.json      [1.]
+n_number_starting_with_dot.json                 [.123]
+n_number_with_leading_zero.json                 [012]
+```
+
+
+
+This test case succeeds with our parser and that's OK since we're
+a token streaming parser (multiple values are allowed):
+```
+n_array_just_minus.json                         [-]
+n_structure_double_array.json                   [][]
+n_structure_whitespace_formfeed.json            [0C] <=> []
+```

runtime/serde/serde-json/common/src/aws/smithy/kotlin/runtime/serde/json/JsonDeserializer.kt

@@ -7,7 +7,7 @@ package aws.smithy.kotlin.runtime.serde.json
 import aws.smithy.kotlin.runtime.serde.*
 
 /**
- * Provides a deserialiser for JSON documents
+ * Provides a deserializer for JSON documents
  *
  * @param payload underlying document from which tokens are read
  */
@@ -65,11 +65,11 @@ class JsonDeserializer(payload: ByteArray) : Deserializer, Deserializer.ElementI
 
     override suspend fun deserializeStruct(descriptor: SdkObjectDescriptor): Deserializer.FieldIterator =
         when (reader.peek()) {
-            RawJsonToken.BeginObject -> {
+            JsonToken.BeginObject -> {
                 reader.nextTokenOf<JsonToken.BeginObject>()
                 JsonFieldIterator(reader, descriptor, this)
             }
-            RawJsonToken.Null -> JsonNullFieldIterator(this)
+            JsonToken.Null -> JsonNullFieldIterator(this)
             else -> throw DeserializationException("Unexpected token type ${reader.peek()}")
         }
 
@@ -88,28 +88,28 @@ class JsonDeserializer(payload: ByteArray) : Deserializer, Deserializer.ElementI
         return token.value
     }
 
-    override suspend fun nextHasValue(): Boolean = reader.peek() != RawJsonToken.Null
+    override suspend fun nextHasValue(): Boolean = reader.peek() != JsonToken.Null
 
     override suspend fun hasNextEntry(): Boolean =
         when (reader.peek()) {
-            RawJsonToken.EndObject -> {
+            JsonToken.EndObject -> {
                 // consume the token
                 reader.nextTokenOf<JsonToken.EndObject>()
                 false
             }
-            RawJsonToken.Null,
-            RawJsonToken.EndDocument -> false
+            JsonToken.Null,
+            JsonToken.EndDocument -> false
             else -> true
         }
 
     override suspend fun hasNextElement(): Boolean =
         when (reader.peek()) {
-            RawJsonToken.EndArray -> {
+            JsonToken.EndArray -> {
                 // consume the token
                 reader.nextTokenOf<JsonToken.EndArray>()
                 false
             }
-            RawJsonToken.EndDocument -> false
+            JsonToken.EndDocument -> false
             else -> true
         }
 }
@@ -131,13 +131,13 @@ private class JsonFieldIterator(
 
     override suspend fun findNextFieldIndex(): Int? {
         val candidate = when (reader.peek()) {
-            RawJsonToken.EndObject -> {
+            JsonToken.EndObject -> {
                 // consume the token
                 reader.nextTokenOf<JsonToken.EndObject>()
                 null
             }
-            RawJsonToken.EndDocument -> null
-            RawJsonToken.Null -> {
+            JsonToken.EndDocument -> null
+            JsonToken.Null -> {
                 reader.nextTokenOf<JsonToken.Null>()
                 null
             }
@@ -151,7 +151,7 @@ private class JsonFieldIterator(
 
         if (candidate != null) {
             // found a field
-            if (reader.peek() == RawJsonToken.Null) {
+            if (reader.peek() == JsonToken.Null) {
                 // skip explicit nulls
                 reader.nextTokenOf<JsonToken.Null>()
                 return findNextFieldIndex()
@@ -166,17 +166,3 @@ private class JsonFieldIterator(
         reader.skipNext()
     }
 }
-
-// return the next token and require that it be of type [TExpected] or else throw an exception
-private suspend inline fun <reified TExpected : JsonToken> JsonStreamReader.nextTokenOf(): TExpected {
-    val token = this.nextToken()
-    requireToken<TExpected>(token)
-    return token as TExpected
-}
-
-// require that the given token be of type [TExpected] or else throw an exception
-private inline fun <reified TExpected> requireToken(token: JsonToken) {
-    if (token::class != TExpected::class) {
-        throw DeserializationException("expected ${TExpected::class}; found ${token::class}")
-    }
-}