Skip to content

Commit e55f807

Browse files
sandwwraithsandwwraith
committed
Test & fix several exception messages from Json parser
To avoid cryptic and incorrect ones, such as `Expected quotation mark '"', but had '"' instead` or `unexpected token: 10`. Fixes #2360 Fixes #2399 Also remove @PublishedApi from BATCH_SIZE to remove it from public API dump.
1 parent 7bf105e commit e55f807

File tree

10 files changed

+217
-33
lines changed

10 files changed

+217
-33
lines changed

‎docs/basic-serialization.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -534,7 +534,7 @@ the `null` value to it.
534534

535535
```text
536536
Exception in thread "main" kotlinx.serialization.json.internal.JsonDecodingException: Unexpected JSON token at offset 52: Expected string literal but 'null' literal was found at path: $.language
537-
Use 'coerceInputValues = true' in 'Json {}` builder to coerce nulls to default values.
537+
Use 'coerceInputValues = true' in 'Json {}' builder to coerce nulls to default values.
538538
```
539539

540540
<!--- TEST LINES_START -->

‎formats/json-tests/commonTest/src/kotlinx/serialization/json/JsonErrorMessagesTest.kt

Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
/*
2+
* Copyright 2017-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
3+
*/
4+
5+
6+
package kotlinx.serialization.json
7+
8+
import kotlinx.serialization.*
9+
import kotlin.test.*
10+
11+
12+
class JsonErrorMessagesTest : JsonTestBase() {
13+
14+
@Test
15+
fun testJsonTokensAreProperlyReported() = parametrizedTest { mode ->
16+
val input1 = """{"boxed":4}"""
17+
val input2 = """{"boxed":"str"}"""
18+
19+
val serString = serializer<Box<String>>()
20+
val serInt = serializer<Box<Int>>()
21+
22+
checkSerializationException({
23+
default.decodeFromString(serString, input1, mode)
24+
}, { message ->
25+
if (mode == JsonTestingMode.TREE)
26+
assertContains(message, "String literal for key 'boxed' should be quoted.")
27+
else
28+
assertContains(
29+
message,
30+
"Unexpected JSON token at offset 9: Expected quotation mark '\"', but had '4' instead at path: \$.boxed"
31+
)
32+
})
33+
34+
checkSerializationException({
35+
default.decodeFromString(serInt, input2, mode)
36+
}, { message ->
37+
if (mode != JsonTestingMode.TREE)
38+
// we allow number values to be quoted, so the message pointing to 's' is correct
39+
assertContains(
40+
message,
41+
"Unexpected JSON token at offset 9: Unexpected symbol 's' in numeric literal at path: \$.boxed"
42+
)
43+
else
44+
assertContains(message, "Failed to parse literal as 'int' value")
45+
})
46+
}
47+
48+
@Test
49+
fun testMissingClosingQuote() = parametrizedTest { mode ->
50+
val input1 = """{"boxed:4}"""
51+
val input2 = """{"boxed":"str}"""
52+
val input3 = """{"boxed:"str"}"""
53+
val serString = serializer<Box<String>>()
54+
val serInt = serializer<Box<Int>>()
55+
56+
checkSerializationException({
57+
default.decodeFromString(serInt, input1, mode)
58+
}, { message ->
59+
// For discussion:
60+
// Technically, both of these messages are correct despite them being completely different.
61+
// A `:` instead of `"` is a good guess, but `:`/`}` is a perfectly valid token inside Json string — for example,
62+
// it can be some kind of path `{"foo:bar:baz":"my:resource:locator:{123}"}` or even URI used as a string key/value.
63+
// So if the closing quote is missing, there's really no way to correctly tell where the key or value is supposed to end.
64+
// Although we may try to unify these messages for consistency.
65+
if (mode in setOf(JsonTestingMode.STREAMING, JsonTestingMode.TREE))
66+
assertContains(
67+
message,
68+
"Unexpected JSON token at offset 7: Expected quotation mark '\"', but had ':' instead at path: \$"
69+
)
70+
else
71+
assertContains(
72+
message, "Unexpected EOF at path: \$"
73+
)
74+
})
75+
76+
checkSerializationException({
77+
default.decodeFromString(serString, input2, mode)
78+
}, { message ->
79+
if (mode in setOf(JsonTestingMode.STREAMING, JsonTestingMode.TREE))
80+
assertContains(
81+
message,
82+
"Unexpected JSON token at offset 13: Expected quotation mark '\"', but had '}' instead at path: \$"
83+
)
84+
else
85+
assertContains(message, "Unexpected EOF at path: \$.boxed")
86+
})
87+
88+
checkSerializationException({
89+
default.decodeFromString(serString, input3, mode)
90+
}, { message ->
91+
assertContains(
92+
message,
93+
"Unexpected JSON token at offset 9: Expected colon ':', but had 's' instead at path: \$"
94+
)
95+
})
96+
}
97+
98+
@Test
99+
fun testUnquoted() = parametrizedTest { mode ->
100+
val input1 = """{boxed:str}"""
101+
val input2 = """{"boxed":str}"""
102+
val ser = serializer<Box<String>>()
103+
104+
checkSerializationException({
105+
default.decodeFromString(ser, input1, mode)
106+
}, { message ->
107+
assertContains(
108+
message,
109+
"""Unexpected JSON token at offset 1: Expected quotation mark '"', but had 'b' instead at path: ${'$'}"""
110+
)
111+
})
112+
113+
checkSerializationException({
114+
default.decodeFromString(ser, input2, mode)
115+
}, { message ->
116+
if (mode == JsonTestingMode.TREE) assertContains(
117+
message,
118+
"""String literal for key 'boxed' should be quoted."""
119+
)
120+
else assertContains(
121+
message,
122+
"""Unexpected JSON token at offset 9: Expected quotation mark '"', but had 's' instead at path: ${'$'}.boxed"""
123+
)
124+
})
125+
}
126+
127+
@Test
128+
fun testNullLiteralForNotNull() = parametrizedTest { mode ->
129+
val input = """{"boxed":null}"""
130+
val ser = serializer<Box<String>>()
131+
checkSerializationException({
132+
default.decodeFromString(ser, input, mode)
133+
}, { message ->
134+
if (mode == JsonTestingMode.TREE)
135+
assertContains(message, "Unexpected 'null' literal when non-nullable string was expected")
136+
else
137+
assertContains(
138+
message,
139+
"Unexpected JSON token at offset 9: Expected string literal but 'null' literal was found at path: \$.boxed"
140+
)
141+
})
142+
}
143+
144+
@Test
145+
fun testEof() = parametrizedTest { mode ->
146+
val input = """{"boxed":"""
147+
checkSerializationException({
148+
default.decodeFromString<Box<String>>(input, mode)
149+
}, { message ->
150+
if (mode == JsonTestingMode.TREE)
151+
assertContains(message, "Cannot read Json element because of unexpected end of the input at path: $")
152+
else
153+
assertContains(message, "Expected quotation mark '\"', but had 'EOF' instead at path: \$.boxed")
154+
155+
})
156+
157+
}
158+
159+
private fun checkSerializationException(action: () -> Unit, assertions: SerializationException.(String) -> Unit) {
160+
val e = assertFailsWith(SerializationException::class, action)
161+
assertNotNull(e.message)
162+
e.assertions(e.message!!)
163+
}
164+
165+
}

‎formats/json-tests/jvmTest/src/kotlinx/serialization/features/JsonSequencePathTest.kt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ class JsonSequencePathTest {
2525
val iterator = Json.decodeToSequence<Data>(source).iterator()
2626
iterator.next() // Ignore
2727
assertFailsWithMessage<SerializationException>(
28-
"Expected quotation mark '\"', but had '2' instead at path: \$.data.s"
28+
"Expected quotation mark '\"', but had '4' instead at path: \$.data.s"
2929
) { iterator.next() }
3030
}
3131

‎formats/json/api/kotlinx-serialization-json.api

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -383,10 +383,6 @@ public final class kotlinx/serialization/json/JvmStreamsKt {
383383
public static final fun encodeToStream (Lkotlinx/serialization/json/Json;Lkotlinx/serialization/SerializationStrategy;Ljava/lang/Object;Ljava/io/OutputStream;)V
384384
}
385385

386-
public final class kotlinx/serialization/json/internal/JsonLexerKt {
387-
public static final field BATCH_SIZE I
388-
}
389-
390386
public final class kotlinx/serialization/json/internal/JsonStreamsKt {
391387
public static final fun decodeByReader (Lkotlinx/serialization/json/Json;Lkotlinx/serialization/DeserializationStrategy;Lkotlinx/serialization/json/internal/SerialReader;)Ljava/lang/Object;
392388
public static final fun decodeToSequenceByReader (Lkotlinx/serialization/json/Json;Lkotlinx/serialization/json/internal/SerialReader;Lkotlinx/serialization/DeserializationStrategy;Lkotlinx/serialization/json/DecodeSequenceMode;)Lkotlin/sequences/Sequence;

‎formats/json/commonMain/src/kotlinx/serialization/json/internal/JsonTreeReader.kt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,7 @@ internal class JsonTreeReader(
101101
result
102102
}
103103
TC_BEGIN_LIST -> readArray()
104-
else -> lexer.fail("Cannot begin reading element, unexpected token: $token")
104+
else -> lexer.fail("Cannot read Json element because of unexpected ${tokenDescription(token)}")
105105
}
106106
}
107107

‎formats/json/commonMain/src/kotlinx/serialization/json/internal/TreeJsonDecoder.kt

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,7 @@ private sealed class AbstractJsonTreeDecoder(
143143
}
144144

145145
private fun unparsedPrimitive(primitive: String): Nothing {
146-
throw JsonDecodingException(-1, "Failed to parse '$primitive'", currentObject().toString())
146+
throw JsonDecodingException(-1, "Failed to parse literal as '$primitive' value", currentObject().toString())
147147
}
148148

149149
override fun decodeTaggedString(tag: String): String {
@@ -159,7 +159,7 @@ private sealed class AbstractJsonTreeDecoder(
159159
}
160160

161161
private fun JsonPrimitive.asLiteral(type: String): JsonLiteral {
162-
return this as? JsonLiteral ?: throw JsonDecodingException(-1, "Unexpected 'null' when $type was expected")
162+
return this as? JsonLiteral ?: throw JsonDecodingException(-1, "Unexpected 'null' literal when non-nullable $type was expected")
163163
}
164164

165165
override fun decodeTaggedInline(tag: String, inlineDescriptor: SerialDescriptor): Decoder =

‎formats/json/commonMain/src/kotlinx/serialization/json/internal/lexer/AbstractJsonLexer.kt

Lines changed: 39 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,8 @@ import kotlin.js.*
1010
import kotlin.jvm.*
1111
import kotlin.math.*
1212

13-
internal const val lenientHint = "Use 'isLenient = true' in 'Json {}` builder to accept non-compliant JSON."
14-
internal const val coerceInputValuesHint = "Use 'coerceInputValues = true' in 'Json {}` builder to coerce nulls to default values."
13+
internal const val lenientHint = "Use 'isLenient = true' in 'Json {}' builder to accept non-compliant JSON."
14+
internal const val coerceInputValuesHint = "Use 'coerceInputValues = true' in 'Json {}' builder to coerce nulls to default values."
1515
internal const val specialFlowingValuesHint =
1616
"It is possible to deserialize them using 'JsonBuilder.allowSpecialFloatingPointValues = true'"
1717
internal const val ignoreUnknownKeysHint = "Use 'ignoreUnknownKeys = true' in 'Json {}' builder to ignore unknown keys."
@@ -56,6 +56,20 @@ private const val ESC2C_MAX = 0x75
5656

5757
internal const val asciiCaseMask = 1 shl 5
5858

59+
internal fun tokenDescription(token: Byte) = when (token) {
60+
TC_STRING -> "quotation mark '\"'"
61+
TC_STRING_ESC -> "string escape sequence '\\'"
62+
TC_COMMA -> "comma ','"
63+
TC_COLON -> "colon ':'"
64+
TC_BEGIN_OBJ -> "start of the object '{'"
65+
TC_END_OBJ -> "end of the object '}'"
66+
TC_BEGIN_LIST -> "start of the array '['"
67+
TC_END_LIST -> "end of the array ']'"
68+
TC_EOF -> "end of the input"
69+
TC_INVALID -> "invalid token"
70+
else -> "valid token" // should never happen
71+
}
72+
5973
// object instead of @SharedImmutable because there is mutual initialization in [initC2ESC] and [initC2TC]
6074
internal object CharMappings {
6175
@JvmField
@@ -200,28 +214,23 @@ internal abstract class AbstractJsonLexer {
200214
}
201215

202216
protected fun unexpectedToken(expected: Char) {
203-
--currentPosition // To properly handle null
204-
if (currentPosition >= 0 && expected == STRING && consumeStringLenient() == NULL) {
205-
fail("Expected string literal but 'null' literal was found", currentPosition - 4, coerceInputValuesHint)
217+
if (currentPosition > 0 && expected == STRING) {
218+
val inputLiteral = withPositionRollback {
219+
currentPosition--
220+
consumeStringLenient()
221+
}
222+
if (inputLiteral == NULL)
223+
fail("Expected string literal but 'null' literal was found", currentPosition - 1, coerceInputValuesHint)
206224
}
207225
fail(charToTokenClass(expected))
208226
}
209227

210-
internal fun fail(expectedToken: Byte): Nothing {
211-
// We know that the token was consumed prior to this call
228+
internal fun fail(expectedToken: Byte, wasConsumed: Boolean = true): Nothing {
212229
// Slow path, never called in normal code, can avoid optimizing it
213-
val expected = when (expectedToken) {
214-
TC_STRING -> "quotation mark '\"'"
215-
TC_COMMA -> "comma ','"
216-
TC_COLON -> "colon ':'"
217-
TC_BEGIN_OBJ -> "start of the object '{'"
218-
TC_END_OBJ -> "end of the object '}'"
219-
TC_BEGIN_LIST -> "start of the array '['"
220-
TC_END_LIST -> "end of the array ']'"
221-
else -> "valid token" // should never happen
222-
}
223-
val s = if (currentPosition == source.length || currentPosition <= 0) "EOF" else source[currentPosition - 1].toString()
224-
fail("Expected $expected, but had '$s' instead", currentPosition - 1)
230+
val expected = tokenDescription(expectedToken)
231+
val position = if (wasConsumed) currentPosition - 1 else currentPosition
232+
val s = if (currentPosition == source.length || position < 0) "EOF" else source[position].toString()
233+
fail("Expected $expected, but had '$s' instead", position)
225234
}
226235

227236
fun peekNextToken(): Byte {
@@ -385,15 +394,15 @@ internal abstract class AbstractJsonLexer {
385394
usedAppend = true
386395
currentPosition = prefetchOrEof(appendEscape(lastPosition, currentPosition))
387396
if (currentPosition == -1)
388-
fail("EOF", currentPosition)
397+
fail("Unexpected EOF", currentPosition)
389398
lastPosition = currentPosition
390399
} else if (++currentPosition >= source.length) {
391400
usedAppend = true
392401
// end of chunk
393402
appendRange(lastPosition, currentPosition)
394403
currentPosition = prefetchOrEof(currentPosition)
395404
if (currentPosition == -1)
396-
fail("EOF", currentPosition)
405+
fail("Unexpected EOF", currentPosition)
397406
lastPosition = currentPosition
398407
}
399408
char = source[currentPosition]
@@ -743,4 +752,13 @@ internal abstract class AbstractJsonLexer {
743752

744753
currentPosition = current + literalSuffix.length
745754
}
755+
756+
private inline fun <T> withPositionRollback(action: () -> T): T {
757+
val snapshot = currentPosition
758+
try {
759+
return action()
760+
} finally {
761+
currentPosition = snapshot
762+
}
763+
}
746764
}

‎formats/json/commonMain/src/kotlinx/serialization/json/internal/lexer/JsonLexer.kt

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@
66

77
package kotlinx.serialization.json.internal
88

9-
@PublishedApi
109
internal const val BATCH_SIZE: Int = 16 * 1024
1110
private const val DEFAULT_THRESHOLD = 128
1211

‎formats/json/commonMain/src/kotlinx/serialization/json/internal/lexer/StringJsonLexer.kt

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@ internal class StringJsonLexer(override val source: String) : AbstractJsonLexer(
7373
if (c == expected) return
7474
unexpectedToken(expected)
7575
}
76+
currentPosition = -1 // for correct EOF reporting
7677
unexpectedToken(expected) // EOF
7778
}
7879

@@ -85,7 +86,12 @@ internal class StringJsonLexer(override val source: String) : AbstractJsonLexer(
8586
consumeNextToken(STRING)
8687
val current = currentPosition
8788
val closingQuote = source.indexOf('"', current)
88-
if (closingQuote == -1) fail(TC_STRING)
89+
if (closingQuote == -1) {
90+
// advance currentPosition to a token after the end of the string to guess position in the error msg
91+
// (not always correct, as `:`/`,` are valid contents of the string, but good guess anyway)
92+
consumeStringLenient()
93+
fail(TC_STRING, wasConsumed = false)
94+
}
8995
// Now we _optimistically_ know where the string ends (it might have been an escaped quote)
9096
for (i in current until closingQuote) {
9197
// Encountered escape sequence, should fallback to "slow" path and symbolic scanning

‎guide/test/BasicSerializationTest.kt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,7 @@ class BasicSerializationTest {
110110
fun testExampleClasses12() {
111111
captureOutput("ExampleClasses12") { example.exampleClasses12.main() }.verifyOutputLinesStart(
112112
"Exception in thread \"main\" kotlinx.serialization.json.internal.JsonDecodingException: Unexpected JSON token at offset 52: Expected string literal but 'null' literal was found at path: $.language",
113-
"Use 'coerceInputValues = true' in 'Json {}` builder to coerce nulls to default values."
113+
"Use 'coerceInputValues = true' in 'Json {}' builder to coerce nulls to default values."
114114
)
115115
}
116116

0 commit comments

Comments
 (0)