Skip to content

Commit 5931d15

Browse files
koperagenkoperagen
committed
Prevent JSON from parsing malformed input
1 parent 0adbb10 commit 5931d15

File tree

8 files changed

+61
-243
lines changed

8 files changed

+61
-243
lines changed

‎core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/readJson.kt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -215,6 +215,7 @@ internal fun fromJsonListAnyColumns(
215215
v.longOrNull != null -> collector.add(v.long)
216216
v.doubleOrNull != null -> collector.add(v.double)
217217
v.floatOrNull != null -> collector.add(v.float)
218+
else -> error("Malformed JSON element ${v::class}: $v")
218219
}
219220
}
220221

@@ -527,6 +528,7 @@ internal fun fromJsonListArrayAndValueColumns(
527528
v.longOrNull != null -> collector.add(v.long)
528529
v.doubleOrNull != null -> collector.add(v.double)
529530
v.floatOrNull != null -> collector.add(v.float)
531+
else -> error("Malformed JSON element ${v::class}: $v")
530532
}
531533
}
532534

‎core/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/CsvTests.kt

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import org.jetbrains.kotlinx.dataframe.testResource
2222
import org.junit.Test
2323
import java.io.File
2424
import java.io.StringWriter
25+
import java.net.URL
2526
import java.util.Locale
2627
import kotlin.reflect.KClass
2728
import kotlin.reflect.typeOf
@@ -249,6 +250,14 @@ class CsvTests {
249250
producedFile.delete()
250251
}
251252

253+
@Test
254+
fun `check integrity of example data`() {
255+
val df = DataFrame.readCSV("../data/jetbrains_repositories.csv")
256+
df.columnNames() shouldBe listOf("full_name", "html_url", "stargazers_count", "topics", "watchers")
257+
df.columnTypes() shouldBe listOf(typeOf<String>(), typeOf<URL>(), typeOf<Int>(), typeOf<String>(), typeOf<Int>())
258+
df shouldBe DataFrame.readCSV("../data/jetbrains repositories.csv")
259+
}
260+
252261
companion object {
253262
private val simpleCsv = testCsv("testCSV")
254263
private val csvWithFrenchLocale = testCsv("testCSVwithFrenchLocale")

‎core/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/json.kt

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,15 @@
11
package org.jetbrains.kotlinx.dataframe.io
22

33
import io.kotest.assertions.throwables.shouldNotThrowAny
4+
import io.kotest.assertions.throwables.shouldThrow
45
import io.kotest.matchers.collections.shouldBeIn
56
import io.kotest.matchers.shouldBe
67
import io.kotest.matchers.string.shouldContain
78
import io.kotest.matchers.string.shouldNotContain
89
import io.kotest.matchers.types.instanceOf
910
import io.kotest.matchers.types.shouldBeInstanceOf
11+
import kotlinx.serialization.json.Json
12+
import kotlinx.serialization.json.JsonElement
1013
import kotlinx.serialization.json.boolean
1114
import kotlinx.serialization.json.int
1215
import kotlinx.serialization.json.jsonArray
@@ -42,6 +45,7 @@ import org.jetbrains.kotlinx.dataframe.impl.io.SerializationKeys.METADATA
4245
import org.jetbrains.kotlinx.dataframe.impl.io.SerializationKeys.NCOL
4346
import org.jetbrains.kotlinx.dataframe.impl.io.SerializationKeys.NROW
4447
import org.jetbrains.kotlinx.dataframe.impl.io.SerializationKeys.VERSION
48+
import org.jetbrains.kotlinx.dataframe.impl.io.readJson
4549
import org.jetbrains.kotlinx.dataframe.impl.nothingType
4650
import org.jetbrains.kotlinx.dataframe.io.JSON.TypeClashTactic.ANY_COLUMNS
4751
import org.jetbrains.kotlinx.dataframe.io.JSON.TypeClashTactic.ARRAY_AND_VALUE_COLUMNS
@@ -1077,4 +1081,13 @@ class JsonTests {
10771081
val json = df.toJson()
10781082
DataFrame.readJsonStr(json) shouldBe df
10791083
}
1084+
1085+
@Test
1086+
fun `parse invalid literal`() {
1087+
// https://github.com/Kotlin/kotlinx.serialization/issues/2511
1088+
val json = Json.decodeFromString<JsonElement>("""[jetbrains, jetbrains-youtrack, youtrack, youtrack-api]""")
1089+
shouldThrow<IllegalStateException> {
1090+
readJson(json, emptyList())
1091+
}
1092+
}
10801093
}

‎plugins/dataframe-gradle-plugin/src/test/kotlin/org/jetbrains/dataframe/gradle/DataFrameReadTest.kt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ class DataFrameReadTest {
3333
fun `file with invalid json`() {
3434
val temp = Files.createTempDirectory("").toFile()
3535
val invalidJson = File(temp, "test.json").also { it.writeText(".") }
36-
shouldNotThrowAny {
36+
shouldThrow<IllegalStateException> {
3737
DataFrame.read(invalidJson)
3838
}
3939
}

‎plugins/kotlin-dataframe/testData/box/read.fir.ir.txt

Lines changed: 12 additions & 92 deletions
Large diffs are not rendered by default.

‎plugins/kotlin-dataframe/testData/box/read.fir.txt

Lines changed: 6 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@ FILE: read.kt
77
@R|org/jetbrains/kotlinx/dataframe/annotations/Order|(order = Int(0)) public abstract val full_name: R|kotlin/String|
88
public get(): R|kotlin/String|
99

10-
@R|org/jetbrains/kotlinx/dataframe/annotations/Order|(order = Int(3)) public abstract val topics: R|org/jetbrains/kotlinx/dataframe/DataFrame<<local>/Topics_391>|
11-
public get(): R|org/jetbrains/kotlinx/dataframe/DataFrame<<local>/Topics_391>|
10+
@R|org/jetbrains/kotlinx/dataframe/annotations/Order|(order = Int(3)) public abstract val topics: R|kotlin/String|
11+
public get(): R|kotlin/String|
1212

1313
@R|org/jetbrains/kotlinx/dataframe/annotations/Order|(order = Int(4)) public abstract val watchers: R|kotlin/Int|
1414
public get(): R|kotlin/Int|
@@ -30,11 +30,11 @@ FILE: read.kt
3030
public final val R|org/jetbrains/kotlinx/dataframe/ColumnsContainer<<local>/Read_16I>|.full_name: R|org/jetbrains/kotlinx/dataframe/DataColumn<kotlin/String>|
3131
public get(): R|org/jetbrains/kotlinx/dataframe/DataColumn<kotlin/String>|
3232

33-
public final val R|org/jetbrains/kotlinx/dataframe/DataRow<<local>/Read_16I>|.topics: R|org/jetbrains/kotlinx/dataframe/DataFrame<<local>/Topics_391>|
34-
public get(): R|org/jetbrains/kotlinx/dataframe/DataFrame<<local>/Topics_391>|
33+
public final val R|org/jetbrains/kotlinx/dataframe/DataRow<<local>/Read_16I>|.topics: R|kotlin/String|
34+
public get(): R|kotlin/String|
3535

36-
public final val R|org/jetbrains/kotlinx/dataframe/ColumnsContainer<<local>/Read_16I>|.topics: R|org/jetbrains/kotlinx/dataframe/DataColumn<org/jetbrains/kotlinx/dataframe/DataFrame<<local>/Topics_391>>|
37-
public get(): R|org/jetbrains/kotlinx/dataframe/DataColumn<org/jetbrains/kotlinx/dataframe/DataFrame<<local>/Topics_391>>|
36+
public final val R|org/jetbrains/kotlinx/dataframe/ColumnsContainer<<local>/Read_16I>|.topics: R|org/jetbrains/kotlinx/dataframe/DataColumn<kotlin/String>|
37+
public get(): R|org/jetbrains/kotlinx/dataframe/DataColumn<kotlin/String>|
3838

3939
public final val R|org/jetbrains/kotlinx/dataframe/DataRow<<local>/Read_16I>|.watchers: R|kotlin/Int|
4040
public get(): R|kotlin/Int|
@@ -58,34 +58,11 @@ FILE: read.kt
5858

5959
}
6060

61-
local abstract class Topics_391 : R|kotlin/Any| {
62-
@R|org/jetbrains/kotlinx/dataframe/annotations/Order|(order = Int(0)) public abstract val value: R|kotlin/Double|
63-
public get(): R|kotlin/Double|
64-
65-
public constructor(): R|<local>/Topics_391|
66-
67-
}
68-
69-
local final class Scope1 : R|kotlin/Any| {
70-
public final val R|org/jetbrains/kotlinx/dataframe/DataRow<<local>/Topics_391>|.value: R|kotlin/Double|
71-
public get(): R|kotlin/Double|
72-
73-
public final val R|org/jetbrains/kotlinx/dataframe/ColumnsContainer<<local>/Topics_391>|.value: R|org/jetbrains/kotlinx/dataframe/DataColumn<kotlin/Double>|
74-
public get(): R|org/jetbrains/kotlinx/dataframe/DataColumn<kotlin/Double>|
75-
76-
public constructor(): R|<local>/Scope1|
77-
78-
}
79-
8061
local abstract class Read_16 : R|<local>/Read_16I| {
8162
public abstract var scope0: R|<local>/Scope0|
8263
public get(): R|<local>/Scope0|
8364
public set(value: R|<local>/Scope0|): R|kotlin/Unit|
8465

85-
public abstract var scope1: R|<local>/Scope1|
86-
public get(): R|<local>/Scope1|
87-
public set(value: R|<local>/Scope1|): R|kotlin/Unit|
88-
8966
public constructor(): R|<local>/Read_16|
9067

9168
}

0 commit comments

Comments
 (0)