Skip to content

Commit fe617a4

Browse files
JolanrensenJolanrensen
committed
fixing tests
1 parent 0f155ae commit fe617a4

File tree

8 files changed

+103
-31
lines changed

8 files changed

+103
-31
lines changed

‎core/build.gradle.kts

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,9 @@ dependencies {
7979
testImplementation(libs.kotlin.scriptingJvm)
8080
testImplementation(libs.jsoup)
8181
testImplementation(libs.sl4jsimple)
82+
testImplementation(projects.dataframeJson)
83+
testImplementation(libs.serialization.core)
84+
testImplementation(libs.serialization.json)
8285

8386
// for samples.api
8487
testImplementation(projects.dataframeCsv)

‎core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/api/parse.kt

Lines changed: 72 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package org.jetbrains.kotlinx.dataframe.impl.api
22

3+
import io.github.oshai.kotlinlogging.KotlinLogging
34
import kotlinx.datetime.Instant
45
import kotlinx.datetime.LocalDate
56
import kotlinx.datetime.LocalDateTime
@@ -10,9 +11,11 @@ import kotlinx.datetime.toKotlinLocalDate
1011
import kotlinx.datetime.toKotlinLocalDateTime
1112
import kotlinx.datetime.toKotlinLocalTime
1213
import org.jetbrains.kotlinx.dataframe.AnyFrame
14+
import org.jetbrains.kotlinx.dataframe.AnyRow
1315
import org.jetbrains.kotlinx.dataframe.ColumnsSelector
1416
import org.jetbrains.kotlinx.dataframe.DataColumn
1517
import org.jetbrains.kotlinx.dataframe.DataFrame
18+
import org.jetbrains.kotlinx.dataframe.DataRow
1619
import org.jetbrains.kotlinx.dataframe.api.GlobalParserOptions
1720
import org.jetbrains.kotlinx.dataframe.api.ParserOptions
1821
import org.jetbrains.kotlinx.dataframe.api.asColumnGroup
@@ -30,7 +33,6 @@ import org.jetbrains.kotlinx.dataframe.columns.size
3033
import org.jetbrains.kotlinx.dataframe.exceptions.TypeConversionException
3134
import org.jetbrains.kotlinx.dataframe.hasNulls
3235
import org.jetbrains.kotlinx.dataframe.impl.api.Parsers.resetToDefault
33-
import org.jetbrains.kotlinx.dataframe.impl.api.Parsers.stringParser
3436
import org.jetbrains.kotlinx.dataframe.impl.canParse
3537
import org.jetbrains.kotlinx.dataframe.impl.catchSilent
3638
import org.jetbrains.kotlinx.dataframe.impl.createStarProjectedType
@@ -60,6 +62,8 @@ import java.time.LocalDate as JavaLocalDate
6062
import java.time.LocalDateTime as JavaLocalDateTime
6163
import java.time.LocalTime as JavaLocalTime
6264

65+
private val logger = KotlinLogging.logger { }
66+
6367
internal interface StringParser<T> {
6468
fun toConverter(options: ParserOptions?): TypeConverter
6569

@@ -210,7 +214,7 @@ internal object Parsers : GlobalParserOptions {
210214
.parseOrNull(this)
211215
?.toInstantUsingOffset()
212216
}
213-
// fallback on the java instant to catch things like "2022-01-23T04:29:60", a.k.a. leap seconds
217+
// fallback on the java instant to catch things like "2022-01-23T04:29:60", a.k.a. leap seconds
214218
?: toJavaInstantOrNull()?.toKotlinInstant()
215219

216220
private fun String.toJavaInstantOrNull(): JavaInstant? =
@@ -405,23 +409,73 @@ internal object Parsers : GlobalParserOptions {
405409
stringParser<BigDecimal> { it.toBigDecimalOrNull() },
406410

407411
// JSON array as DataFrame<*> TODO
408-
// stringParser<AnyFrame>(catch = true) {
409-
// val trimmed = it.trim()
410-
// if (trimmed.startsWith("[") && trimmed.endsWith("]")) {
411-
// DataFrame.readJsonStr(it)
412-
// } else {
413-
// null
414-
// }
415-
// },
412+
stringParser<AnyFrame>(catch = true) {
413+
val trimmed = it.trim()
414+
if (trimmed.startsWith("[") && trimmed.endsWith("]")) {
415+
try {
416+
val klass = Class.forName("org.jetbrains.kotlinx.dataframe.io.JsonKt")
417+
val typeClashTactic = Class.forName("org.jetbrains.kotlinx.dataframe.io.JSON\$TypeClashTactic")
418+
val readJsonStr = klass.getMethod(
419+
"readJsonStr",
420+
/* this = */DataFrame.Companion::class.java,
421+
/* text = */ String::class.java,
422+
/* header = */ List::class.java,
423+
/* keyValuePaths = */ List::class.java,
424+
/* typeClashTactic = */ typeClashTactic,
425+
/* unifyNumbers = */ Boolean::class.java,
426+
)
427+
428+
readJsonStr.invoke(
429+
null,
430+
/* this = */ DataFrame.Companion,
431+
/* text = */ trimmed,
432+
/* header = */ emptyList<Any>(),
433+
/* keyValuePaths = */ emptyList<Any>(),
434+
/* typeClashTactic = */ typeClashTactic.enumConstants[0],
435+
/* unifyNumbers = */ true,
436+
) as AnyFrame
437+
} catch (_: ClassNotFoundException) {
438+
logger.warn { "parse() encountered a string that looks like a JSON array, but the dataframe-json dependency was not detected. Skipping for now." }
439+
null
440+
}
441+
} else {
442+
null
443+
}
444+
},
416445
// JSON object as DataRow<*> TODO
417-
// stringParser<AnyRow>(catch = true) {
418-
// val trimmed = it.trim()
419-
// if (trimmed.startsWith("{") && trimmed.endsWith("}")) {
420-
// DataRow.readJsonStr(it)
421-
// } else {
422-
// null
423-
// }
424-
// },
446+
stringParser<AnyRow>(catch = true) {
447+
val trimmed = it.trim()
448+
if (trimmed.startsWith("{") && trimmed.endsWith("}")) {
449+
try {
450+
val klass = Class.forName("org.jetbrains.kotlinx.dataframe.io.JsonKt")
451+
val typeClashTactic = Class.forName("org.jetbrains.kotlinx.dataframe.io.JSON\$TypeClashTactic")
452+
val readJsonStr = klass.getMethod(
453+
"readJsonStr",
454+
/* this = */DataRow.Companion::class.java,
455+
/* text = */ String::class.java,
456+
/* header = */ List::class.java,
457+
/* keyValuePaths = */ List::class.java,
458+
/* typeClashTactic = */ typeClashTactic,
459+
/* unifyNumbers = */ Boolean::class.java,
460+
)
461+
462+
readJsonStr.invoke(
463+
null,
464+
/* this = */ DataRow.Companion,
465+
/* text = */ trimmed,
466+
/* header = */ emptyList<Any>(),
467+
/* keyValuePaths = */ emptyList<Any>(),
468+
/* typeClashTactic = */ typeClashTactic.enumConstants[0],
469+
/* unifyNumbers = */ true,
470+
) as AnyRow
471+
} catch (_: ClassNotFoundException) {
472+
logger.warn { "parse() encountered a string that looks like a JSON object, but the dataframe-json dependency was not detected. Skipping for now." }
473+
null
474+
}
475+
} else {
476+
null
477+
}
478+
},
425479
// Char
426480
stringParser<Char> { it.singleOrNull() },
427481
// No parser found, return as String

‎core/src/test/kotlin/org/jetbrains/kotlinx/dataframe/Utils.kt

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,5 +31,3 @@ fun <T : DataFrame<*>> T.alsoDebug(println: String? = null, rowsLimit: Int = 20)
3131
print(borders = true, title = true, columnTypes = true, valueLimit = -1, rowsLimit = rowsLimit)
3232
schema().print()
3333
}
34-
35-
fun parseJsonStr(jsonStr: String): JsonObject = Json.parseToJsonElement(jsonStr).jsonObject

‎dataframe-csv/build.gradle.kts

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,6 @@ dependencies {
4242
testImplementation(libs.kotlinx.benchmark.runtime)
4343
testImplementation(libs.junit)
4444
testImplementation(libs.sl4jsimple)
45-
testImplementation(projects.dataframeJson)
4645
testImplementation(libs.kotestAssertions) {
4746
exclude("org.jetbrains.kotlin", "kotlin-stdlib-jdk8")
4847
}

‎dataframe-json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/writeJson.kt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -426,7 +426,7 @@ private fun isDataframeConvertable(dataframeLike: Any?) =
426426
KotlinNotebookPluginUtils.isDataframeConvertable(dataframeLike = dataframeLike)
427427

428428
@Suppress("INVISIBLE_REFERENCE")
429-
private fun BufferedImage.resizeKeepingAspectRatio(
429+
internal fun BufferedImage.resizeKeepingAspectRatio(
430430
maxSize: Int,
431431
resultImageType: Int = BufferedImage.TYPE_INT_ARGB,
432432
interpolation: Any = RenderingHints.VALUE_INTERPOLATION_NEAREST_NEIGHBOR,

‎dataframe-json/src/main/resources/services/org.jetbrains.kotlinx.dataframe.io.SupportedFormat renamed to ‎dataframe-json/src/main/resources/META-INF/services/org.jetbrains.kotlinx.dataframe.io.SupportedFormat

File renamed without changes.

‎core/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/ImageSerializationTests.kt renamed to ‎dataframe-json/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/ImageSerializationTests.kt

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,6 @@ import org.jetbrains.kotlinx.dataframe.impl.io.resizeKeepingAspectRatio
1313
import org.jetbrains.kotlinx.dataframe.io.Base64ImageEncodingOptions.Companion.ALL_OFF
1414
import org.jetbrains.kotlinx.dataframe.io.Base64ImageEncodingOptions.Companion.GZIP_ON
1515
import org.jetbrains.kotlinx.dataframe.io.Base64ImageEncodingOptions.Companion.LIMIT_SIZE_ON
16-
import org.jetbrains.kotlinx.dataframe.parseJsonStr
17-
import org.jetbrains.kotlinx.dataframe.testResource
1816
import org.junit.Test
1917
import org.junit.runner.RunWith
2018
import org.junit.runners.Parameterized
@@ -25,6 +23,7 @@ import java.io.File
2523
import java.util.Base64
2624
import java.util.zip.GZIPInputStream
2725
import javax.imageio.ImageIO
26+
import kotlin.math.abs
2827

2928
@RunWith(Parameterized::class)
3029
class ImageSerializationTests(private val encodingOptions: Base64ImageEncodingOptions?) {
@@ -145,7 +144,7 @@ class ImageSerializationTests(private val encodingOptions: Base64ImageEncodingOp
145144
val g2 = (rgb2 shr 8) and 0xFF
146145
val b2 = rgb2 and 0xFF
147146

148-
val diff = kotlin.math.abs(r1 - r2) + kotlin.math.abs(g1 - g2) + kotlin.math.abs(b1 - b2)
147+
val diff = abs(r1 - r2) + abs(g1 - g2) + abs(b1 - b2)
149148

150149
// If the difference in color components exceed our allowance return false
151150
if (diff > allowedDelta) {

‎core/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/json.kt renamed to ‎dataframe-json/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/json.kt

Lines changed: 25 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import io.kotest.matchers.types.instanceOf
1010
import io.kotest.matchers.types.shouldBeInstanceOf
1111
import kotlinx.serialization.json.Json
1212
import kotlinx.serialization.json.JsonElement
13+
import kotlinx.serialization.json.JsonObject
1314
import kotlinx.serialization.json.boolean
1415
import kotlinx.serialization.json.int
1516
import kotlinx.serialization.json.jsonArray
@@ -19,7 +20,6 @@ import org.intellij.lang.annotations.Language
1920
import org.jetbrains.kotlinx.dataframe.AnyFrame
2021
import org.jetbrains.kotlinx.dataframe.DataFrame
2122
import org.jetbrains.kotlinx.dataframe.DataRow
22-
import org.jetbrains.kotlinx.dataframe.alsoDebug
2323
import org.jetbrains.kotlinx.dataframe.api.JsonPath
2424
import org.jetbrains.kotlinx.dataframe.api.allNulls
2525
import org.jetbrains.kotlinx.dataframe.api.colsOf
@@ -29,6 +29,7 @@ import org.jetbrains.kotlinx.dataframe.api.dataFrameOf
2929
import org.jetbrains.kotlinx.dataframe.api.forEach
3030
import org.jetbrains.kotlinx.dataframe.api.getColumnGroup
3131
import org.jetbrains.kotlinx.dataframe.api.getFrameColumn
32+
import org.jetbrains.kotlinx.dataframe.api.print
3233
import org.jetbrains.kotlinx.dataframe.api.schema
3334
import org.jetbrains.kotlinx.dataframe.api.toFloat
3435
import org.jetbrains.kotlinx.dataframe.api.toMap
@@ -46,16 +47,14 @@ import org.jetbrains.kotlinx.dataframe.impl.io.SerializationKeys.NCOL
4647
import org.jetbrains.kotlinx.dataframe.impl.io.SerializationKeys.NROW
4748
import org.jetbrains.kotlinx.dataframe.impl.io.SerializationKeys.VERSION
4849
import org.jetbrains.kotlinx.dataframe.impl.io.readJsonImpl
49-
import org.jetbrains.kotlinx.dataframe.impl.nothingType
50-
import org.jetbrains.kotlinx.dataframe.impl.nullableNothingType
5150
import org.jetbrains.kotlinx.dataframe.io.JSON.TypeClashTactic.ANY_COLUMNS
5251
import org.jetbrains.kotlinx.dataframe.io.JSON.TypeClashTactic.ARRAY_AND_VALUE_COLUMNS
53-
import org.jetbrains.kotlinx.dataframe.parseJsonStr
54-
import org.jetbrains.kotlinx.dataframe.testJson
5552
import org.jetbrains.kotlinx.dataframe.type
5653
import org.jetbrains.kotlinx.dataframe.values
5754
import org.junit.Test
55+
import java.net.URL
5856
import kotlin.Double
57+
import kotlin.reflect.KType
5958
import kotlin.reflect.typeOf
6059

6160
@Suppress("ktlint:standard:argument-list-wrapping")
@@ -1121,7 +1120,7 @@ class JsonTests {
11211120
@Test
11221121
fun `serialize column with list of objects`() {
11231122
val df = dataFrameOf("col")(Regex(".+").findAll("abc").toList())
1124-
val json = shouldNotThrowAny { df.toJson() }!!
1123+
val json = shouldNotThrowAny { df.toJson() }
11251124
val list = DataFrame.readJsonStr(json)["col"][0].shouldBeInstanceOf<List<*>>()
11261125
list[0].shouldBeInstanceOf<String>()
11271126
}
@@ -1142,3 +1141,23 @@ class JsonTests {
11421141
}
11431142
}
11441143
}
1144+
1145+
fun testResource(resourcePath: String): URL = JsonTests::class.java.classLoader.getResource(resourcePath)!!
1146+
1147+
fun parseJsonStr(jsonStr: String): JsonObject = Json.parseToJsonElement(jsonStr).jsonObject
1148+
1149+
fun testJson(jsonName: String) = testResource("$jsonName.json")
1150+
1151+
/**
1152+
* Prints dataframe to console with borders, title, column types and schema
1153+
*/
1154+
fun <T : DataFrame<*>> T.alsoDebug(println: String? = null, rowsLimit: Int = 20): T =
1155+
apply {
1156+
println?.let { println(it) }
1157+
print(borders = true, title = true, columnTypes = true, valueLimit = -1, rowsLimit = rowsLimit)
1158+
schema().print()
1159+
}
1160+
1161+
internal val nothingType: KType = typeOf<List<Nothing>>().arguments.first().type!!
1162+
internal val nullableNothingType: KType = typeOf<List<Nothing?>>().arguments.first().type!!
1163+
internal fun nothingType(nullable: Boolean): KType = if (nullable) nullableNothingType else nothingType

0 commit comments

Comments
 (0)