Skip to content

Commit d061d13

Browse files
devcrocoddevcrocod
committed
Resolve conflicts after #573 pr
1 parent b7e4558 commit d061d13

File tree

9 files changed

+312
-951
lines changed

9 files changed

+312
-951
lines changed

‎core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/readJson.kt

Lines changed: 87 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,21 @@
11
package org.jetbrains.kotlinx.dataframe.impl.io
22

3-
import com.beust.klaxon.JsonArray
4-
import com.beust.klaxon.JsonObject
3+
import kotlinx.serialization.json.JsonArray
4+
import kotlinx.serialization.json.JsonNull
5+
import kotlinx.serialization.json.JsonObject
6+
import kotlinx.serialization.json.JsonPrimitive
7+
import kotlinx.serialization.json.boolean
8+
import kotlinx.serialization.json.booleanOrNull
9+
import kotlinx.serialization.json.double
10+
import kotlinx.serialization.json.doubleOrNull
11+
import kotlinx.serialization.json.float
12+
import kotlinx.serialization.json.floatOrNull
13+
import kotlinx.serialization.json.int
14+
import kotlinx.serialization.json.intOrNull
15+
import kotlinx.serialization.json.jsonArray
16+
import kotlinx.serialization.json.jsonPrimitive
17+
import kotlinx.serialization.json.long
18+
import kotlinx.serialization.json.longOrNull
519
import org.jetbrains.kotlinx.dataframe.AnyCol
620
import org.jetbrains.kotlinx.dataframe.AnyFrame
721
import org.jetbrains.kotlinx.dataframe.DataColumn
@@ -73,8 +87,8 @@ internal fun readJson(
7387
val df: AnyFrame = when (typeClashTactic) {
7488
ARRAY_AND_VALUE_COLUMNS -> {
7589
when (parsed) {
76-
is JsonArray<*> -> fromJsonListArrayAndValueColumns(
77-
records = parsed.value,
90+
is JsonArray -> fromJsonListArrayAndValueColumns(
91+
records = parsed,
7892
header = header,
7993
keyValuePaths = keyValuePaths,
8094
)
@@ -88,8 +102,8 @@ internal fun readJson(
88102

89103
ANY_COLUMNS -> {
90104
when (parsed) {
91-
is JsonArray<*> -> fromJsonListAnyColumns(
92-
records = parsed.value,
105+
is JsonArray -> fromJsonListAnyColumns(
106+
records = parsed,
93107
header = header,
94108
keyValuePaths = keyValuePaths,
95109
)
@@ -126,18 +140,16 @@ internal fun fromJsonListAnyColumns(
126140

127141
// list element type can be JsonObject, JsonArray or primitive
128142
val nameGenerator = ColumnNameGenerator()
129-
records.forEach {
130-
when (it) {
143+
records.forEach { record ->
144+
when (record) {
131145
is JsonObject -> {
132146
hasObject = true
133-
it.entries.forEach {
134-
nameGenerator.addIfAbsent(it.key)
135-
}
147+
record.entries.forEach { nameGenerator.addIfAbsent(it.key) }
136148
}
137149

138-
is JsonArray<*> -> hasArray = true
139-
null -> Unit
140-
else -> hasPrimitive = true
150+
is JsonArray -> hasArray = true
151+
is JsonNull, null -> Unit
152+
is JsonPrimitive -> hasPrimitive = true
141153
}
142154
}
143155

@@ -155,7 +167,7 @@ internal fun fromJsonListAnyColumns(
155167

156168
@Suppress("KotlinConstantConditions")
157169
val columns: List<AnyCol> = when {
158-
// Create one column of type Any? (or guessed primitive type) from all the records
170+
// Create one column of type Any? (or guessed a primitive type) from all the records
159171
colType == AnyColType.ANY -> {
160172
val collector: DataCollectorBase<Any?> =
161173
if (justPrimitives) createDataCollector(records.size) // guess the type
@@ -177,7 +189,7 @@ internal fun fromJsonListAnyColumns(
177189
)
178190
}
179191

180-
is JsonArray<*> -> {
192+
is JsonArray -> {
181193
val parsed = fromJsonListAnyColumns(
182194
records = v,
183195
keyValuePaths = keyValuePaths,
@@ -189,9 +201,21 @@ internal fun fromJsonListAnyColumns(
189201
)
190202
}
191203

192-
"NaN" -> {
193-
nanIndices.add(i)
194-
collector.add(null)
204+
is JsonPrimitive -> {
205+
when {
206+
v.content == "NaN" -> {
207+
nanIndices.add(i)
208+
collector.add(null)
209+
}
210+
211+
v.isString -> collector.add(v.content)
212+
v.booleanOrNull != null -> collector.add(v.boolean)
213+
v.intOrNull != null -> collector.add(v.int)
214+
v.longOrNull != null -> collector.add(v.long)
215+
v.doubleOrNull != null -> collector.add(v.double)
216+
v.floatOrNull != null -> collector.add(v.float)
217+
v.jsonPrimitive is JsonNull -> collector.add(null)
218+
}
195219
}
196220

197221
else -> collector.add(v)
@@ -227,8 +251,8 @@ internal fun fromJsonListAnyColumns(
227251
records.forEach {
228252
startIndices.add(values.size)
229253
when (it) {
230-
is JsonArray<*> -> values.addAll(it.value)
231-
null -> Unit
254+
is JsonArray -> values.addAll(it)
255+
is JsonNull, null -> Unit
232256
else -> error("Expected JsonArray, got $it")
233257
}
234258
}
@@ -242,10 +266,10 @@ internal fun fromJsonListAnyColumns(
242266
parsed.isSingleUnnamedColumn() -> {
243267
val col = (parsed.getColumn(0) as UnnamedColumn).col
244268
val elementType = col.type
245-
val values = col.values.asList().splitByIndices(startIndices.asSequence()).toList()
269+
val columnValues = col.values.asList().splitByIndices(startIndices.asSequence()).toList()
246270
DataColumn.createValueColumn(
247271
name = arrayColumnName,
248-
values = values,
272+
values = columnValues,
249273
type = List::class.createType(listOf(KTypeProjection.invariant(elementType))),
250274
)
251275
}
@@ -263,10 +287,10 @@ internal fun fromJsonListAnyColumns(
263287
colType == AnyColType.OBJECTS && isKeyValue -> {
264288
// collect the value types to make sure Value columns with lists and other values aren't all turned into lists
265289
val valueTypes = mutableSetOf<KType>()
266-
val dataFrames = records.map {
267-
when (it) {
290+
val dataFrames = records.map { record ->
291+
when (record) {
268292
is JsonObject -> {
269-
val map = it.map.mapValues { (key, value) ->
293+
val map = record.mapValues { (key, value) ->
270294
val parsed = fromJsonListAnyColumns(
271295
records = listOf(value),
272296
keyValuePaths = keyValuePaths,
@@ -288,8 +312,8 @@ internal fun fromJsonListAnyColumns(
288312
)
289313
}
290314

291-
null -> DataFrame.emptyOf<AnyKeyValueProperty>()
292-
else -> error("Expected JsonObject, got $it")
315+
is JsonNull, null -> DataFrame.emptyOf<AnyKeyValueProperty>()
316+
else -> error("Expected JsonObject, got $record")
293317
}
294318
}
295319

@@ -328,7 +352,7 @@ internal fun fromJsonListAnyColumns(
328352
records.forEach {
329353
when (it) {
330354
is JsonObject -> values.add(it[colName])
331-
null -> values.add(null)
355+
is JsonNull, null -> values.add(null)
332356
else -> error("Expected JsonObject, got $it")
333357
}
334358
}
@@ -395,24 +419,24 @@ internal fun fromJsonListArrayAndValueColumns(
395419

396420
// list element type can be JsonObject, JsonArray or primitive
397421
// So first, we gather all properties of objects to merge including "array" and "value" if needed
398-
// so the resulting type of a property with instances 123, ["abc"], and { "a": 1, "b": 2 } will be
422+
// so the resulting type of property with instances 123, ["abc"], and { "a": 1, "b": 2 } will be
399423
// { array: List<String>, value: Int?, a: Int?, b: Int? }
400424
// and instances will look like
401425
// { "array": [], "value": 123, "a": null, "b": null }
402426

403427
val nameGenerator = ColumnNameGenerator()
404-
records.forEach {
405-
when (it) {
406-
is JsonObject -> it.entries.forEach {
428+
records.forEach { record ->
429+
when (record) {
430+
is JsonObject -> record.entries.forEach {
407431
nameGenerator.addIfAbsent(it.key)
408432
}
409433

410-
is JsonArray<*> -> hasArray = true
411-
null -> Unit
412-
else -> hasPrimitive = true
434+
is JsonArray -> hasArray = true
435+
is JsonNull, null -> Unit
436+
is JsonPrimitive -> hasPrimitive = true
413437
}
414438
}
415-
if (records.all { it == null }) hasPrimitive = true
439+
if (records.all { it == null || it is JsonNull }) hasPrimitive = true
416440

417441
// Add a value column to the collected names if needed
418442
val valueColumn = if (hasPrimitive || records.isEmpty()) {
@@ -433,10 +457,10 @@ internal fun fromJsonListArrayAndValueColumns(
433457
val columns: List<AnyCol> = when {
434458
// instead of using the names, generate a single key/value frame column
435459
isKeyValue -> {
436-
val dataFrames = records.map {
437-
when (it) {
460+
val dataFrames = records.map { record ->
461+
when (record) {
438462
is JsonObject -> {
439-
val map = it.map.mapValues { (key, value) ->
463+
val map = record.mapValues { (key, value) ->
440464
val parsed = fromJsonListArrayAndValueColumns(
441465
records = listOf(value),
442466
keyValuePaths = keyValuePaths,
@@ -459,8 +483,8 @@ internal fun fromJsonListArrayAndValueColumns(
459483
)
460484
}
461485

462-
null -> DataFrame.emptyOf<AnyKeyValueProperty>()
463-
else -> error("Expected JsonObject, got $it")
486+
is JsonNull, null -> DataFrame.emptyOf<AnyKeyValueProperty>()
487+
else -> error("Expected JsonObject, got $record")
464488
}
465489
}
466490

@@ -488,10 +512,23 @@ internal fun fromJsonListArrayAndValueColumns(
488512
records.forEachIndexed { i, v ->
489513
when (v) {
490514
is JsonObject -> collector.add(null)
491-
is JsonArray<*> -> collector.add(null)
492-
"NaN" -> {
493-
nanIndices.add(i)
494-
collector.add(null)
515+
is JsonArray -> collector.add(null)
516+
is JsonPrimitive -> {
517+
when {
518+
v.content == "NaN" -> {
519+
nanIndices.add(i)
520+
collector.add(null)
521+
}
522+
523+
v.isString -> collector.add(v.content)
524+
v.booleanOrNull != null -> collector.add(v.boolean)
525+
v.intOrNull != null -> collector.add(v.int)
526+
v.longOrNull != null -> collector.add(v.long)
527+
v.doubleOrNull != null -> collector.add(v.double)
528+
v.floatOrNull != null -> collector.add(v.float)
529+
v is JsonNull -> collector.add(null)
530+
else -> collector.add(v)
531+
}
495532
}
496533

497534
else -> collector.add(v)
@@ -526,7 +563,7 @@ internal fun fromJsonListArrayAndValueColumns(
526563
val startIndices = ArrayList<Int>()
527564
records.forEach {
528565
startIndices.add(values.size)
529-
if (it is JsonArray<*>) values.addAll(it.value)
566+
if (it is JsonArray) values.addAll(it.jsonArray)
530567
}
531568
val parsed = fromJsonListArrayAndValueColumns(
532569
records = values,
@@ -538,10 +575,11 @@ internal fun fromJsonListArrayAndValueColumns(
538575
parsed.isSingleUnnamedColumn() -> {
539576
val col = (parsed.getColumn(0) as UnnamedColumn).col
540577
val elementType = col.type
541-
val values = col.values.asList().splitByIndices(startIndices.asSequence()).toList()
578+
val columnValues =
579+
col.values.asList().splitByIndices(startIndices.asSequence()).toList()
542580
DataColumn.createValueColumn(
543581
name = colName,
544-
values = values,
582+
values = columnValues,
545583
type = List::class.createType(listOf(KTypeProjection.invariant(elementType))),
546584
)
547585
}

0 commit comments

Comments
 (0)