Skip to content

Commit 341b7c8

Browse files
JolanrensenJolanrensen
committed
fully decoupled json module, made it optional for csv and excel using reflection
1 parent 8ed6fe6 commit 341b7c8

File tree

18 files changed

+121
-44
lines changed

18 files changed

+121
-44
lines changed

‎core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/annotations/ImportDataSchema.kt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ public annotation class JdbcOptions(
7474
public annotation class JsonOptions(
7575
/**
7676
* Allows the choice of how to handle type clashes when reading a JSON file.
77-
* Must be either [TypeClashTactics.ARRAY_AND_VALUE_COLUMNS] or [TypeClashTactics.ANY_COLUMNS]
77+
* Must be either [JsonOptions.TypeClashTactics.ARRAY_AND_VALUE_COLUMNS] or [JsonOptions.TypeClashTactics.ANY_COLUMNS]
7878
* */
7979
public val typeClashTactic: String = TypeClashTactics.ARRAY_AND_VALUE_COLUMNS,
8080
/**

‎core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/api/parse.kt

Lines changed: 21 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,9 @@ import kotlinx.datetime.toKotlinLocalDate
1010
import kotlinx.datetime.toKotlinLocalDateTime
1111
import kotlinx.datetime.toKotlinLocalTime
1212
import org.jetbrains.kotlinx.dataframe.AnyFrame
13-
import org.jetbrains.kotlinx.dataframe.AnyRow
1413
import org.jetbrains.kotlinx.dataframe.ColumnsSelector
1514
import org.jetbrains.kotlinx.dataframe.DataColumn
1615
import org.jetbrains.kotlinx.dataframe.DataFrame
17-
import org.jetbrains.kotlinx.dataframe.DataRow
1816
import org.jetbrains.kotlinx.dataframe.api.GlobalParserOptions
1917
import org.jetbrains.kotlinx.dataframe.api.ParserOptions
2018
import org.jetbrains.kotlinx.dataframe.api.asColumnGroup
@@ -31,13 +29,14 @@ import org.jetbrains.kotlinx.dataframe.columns.TypeSuggestion
3129
import org.jetbrains.kotlinx.dataframe.columns.size
3230
import org.jetbrains.kotlinx.dataframe.exceptions.TypeConversionException
3331
import org.jetbrains.kotlinx.dataframe.hasNulls
32+
import org.jetbrains.kotlinx.dataframe.impl.api.Parsers.resetToDefault
33+
import org.jetbrains.kotlinx.dataframe.impl.api.Parsers.stringParser
3434
import org.jetbrains.kotlinx.dataframe.impl.canParse
3535
import org.jetbrains.kotlinx.dataframe.impl.catchSilent
3636
import org.jetbrains.kotlinx.dataframe.impl.createStarProjectedType
3737
import org.jetbrains.kotlinx.dataframe.impl.io.FastDoubleParser
3838
import org.jetbrains.kotlinx.dataframe.impl.javaDurationCanParse
3939
import org.jetbrains.kotlinx.dataframe.io.isUrl
40-
import org.jetbrains.kotlinx.dataframe.io.readJsonStr
4140
import org.jetbrains.kotlinx.dataframe.values
4241
import java.math.BigDecimal
4342
import java.math.BigInteger
@@ -404,24 +403,25 @@ internal object Parsers : GlobalParserOptions {
404403
stringParser<BigInteger> { it.toBigIntegerOrNull() },
405404
// BigDecimal
406405
stringParser<BigDecimal> { it.toBigDecimalOrNull() },
407-
// JSON array as DataFrame<*>
408-
stringParser<AnyFrame>(catch = true) {
409-
val trimmed = it.trim()
410-
if (trimmed.startsWith("[") && trimmed.endsWith("]")) {
411-
DataFrame.readJsonStr(it)
412-
} else {
413-
null
414-
}
415-
},
416-
// JSON object as DataRow<*>
417-
stringParser<AnyRow>(catch = true) {
418-
val trimmed = it.trim()
419-
if (trimmed.startsWith("{") && trimmed.endsWith("}")) {
420-
DataRow.readJsonStr(it)
421-
} else {
422-
null
423-
}
424-
},
406+
407+
// JSON array as DataFrame<*> TODO
408+
// stringParser<AnyFrame>(catch = true) {
409+
// val trimmed = it.trim()
410+
// if (trimmed.startsWith("[") && trimmed.endsWith("]")) {
411+
// DataFrame.readJsonStr(it)
412+
// } else {
413+
// null
414+
// }
415+
// },
416+
// JSON object as DataRow<*> TODO
417+
// stringParser<AnyRow>(catch = true) {
418+
// val trimmed = it.trim()
419+
// if (trimmed.startsWith("{") && trimmed.endsWith("}")) {
420+
// DataRow.readJsonStr(it)
421+
// } else {
422+
// null
423+
// }
424+
// },
425425
// Char
426426
stringParser<Char> { it.singleOrNull() },
427427
// No parser found, return as String

‎core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/image.kt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ internal fun BufferedImage.toByteArray(format: String = DEFAULT_IMG_FORMAT): Byt
6060
}
6161

6262
// helper overload for friend modules
63+
@JvmName("resizeKeepingAspectRatioOverload")
6364
internal fun resizeKeepingAspectRatio(
6465
image: BufferedImage,
6566
maxSize: Int,
@@ -78,4 +79,5 @@ internal fun resizeKeepingAspectRatio(
7879
)
7980

8081
// helper overload for friend modules
82+
@JvmName("toByteArrayOverload")
8183
internal fun toByteArray(image: BufferedImage, format: String = DEFAULT_IMG_FORMAT) = image.toByteArray(format)

‎core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/schema/Utils.kt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ internal fun AnyFrame.extractSchema(): DataFrameSchema =
3030
DataFrameSchemaImpl(columns().filter { it.name().isNotEmpty() }.associate { it.name() to it.extractSchema() })
3131

3232
// helper overload for friend modules
33+
@JvmName("intersectSchemasOverload")
3334
internal fun intersectSchemas(schemas: Iterable<DataFrameSchema>): DataFrameSchema = schemas.intersectSchemas()
3435

3536
internal fun Iterable<DataFrameSchema>.intersectSchemas(): DataFrameSchema {

‎core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/common.kt

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ import java.net.URL
1717
* Opens a stream to [url] to create a [DataFrame] from it.
1818
* If the URL is a file URL, the file is read directly.
1919
* If the URL is an HTTP URL, it's also read directly, but if the server returns an error code,
20-
* the error response is read as JSON and parsed as [DataFrame] too.
20+
* the error response is read and parsed as [DataFrame] too.
2121
*
2222
* Public so it may be used in other modules.
2323
*/
@@ -32,8 +32,8 @@ public fun catchHttpResponse(url: URL, body: (InputStream) -> AnyFrame): AnyFram
3232
if (code != 200) {
3333
val response = connection.responseMessage
3434
try {
35-
// attempt to read error response as JSON
36-
return DataFrame.readJson(connection.errorStream)
35+
// attempt to read error response as dataframe
36+
return DataFrame.read(connection.errorStream).df
3737
} catch (_: Exception) {
3838
throw RuntimeException("Server returned HTTP response code: $code. Response: $response")
3939
}

‎core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/csv.kt

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -430,9 +430,17 @@ public fun AnyFrame.writeCSV(writer: Appendable, format: CSVFormat = CSVFormat.D
430430
}
431431
forEach {
432432
val values = it.values.map {
433-
when (it) {
434-
is AnyRow -> it.toJson()
435-
is AnyFrame -> it.toJson()
433+
when (it) { // todo use compileOnly?
434+
is AnyRow ->
435+
error(
436+
"Encountered a DataRow when writing CSV. This needs to be converted to JSON, which is not supported by `writeCSV` anymore. Please use `df.writeCsv()` instead.",
437+
)
438+
439+
is AnyFrame ->
440+
error(
441+
"Encountered a DataFrame when writing CSV. This needs to be converted to JSON, which is not supported by `writeCSV` anymore. Please use `df.writeCsv()` instead.",
442+
)
443+
436444
else -> it
437445
}
438446
}

‎dataframe-csv/build.gradle.kts

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,13 +34,11 @@ dependencies {
3434
implementation(libs.kotlinLogging)
3535
implementation(libs.kotlin.reflect)
3636

37-
// for writing json in csv
38-
implementation(projects.dataframeJson)
39-
4037
testApi(projects.core)
4138
testImplementation(libs.kotlinx.benchmark.runtime)
4239
testImplementation(libs.junit)
4340
testImplementation(libs.sl4jsimple)
41+
testImplementation(projects.dataframeJson)
4442
testImplementation(libs.kotestAssertions) {
4543
exclude("org.jetbrains.kotlin", "kotlin-stdlib-jdk8")
4644
}

‎dataframe-csv/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/jsonHelper.kt

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
package org.jetbrains.kotlinx.dataframe.impl.io
2+
3+
import org.jetbrains.kotlinx.dataframe.AnyFrame
4+
import org.jetbrains.kotlinx.dataframe.AnyRow
5+
6+
internal fun AnyFrame.toJson(prettyPrint: Boolean = false): String {
7+
val jsonClass = try {
8+
Class.forName("org.jetbrains.kotlinx.dataframe.io.JsonKt")
9+
} catch (_: ClassNotFoundException) {
10+
error(
11+
"Encountered a DataFrame when writing to csv/tsv/delim. This needs to be converted to JSON, so the dataframe-json dependency is required.",
12+
)
13+
}
14+
return jsonClass.getMethod("toJson", AnyFrame::class.java, Boolean::class.java)
15+
.invoke(null, this, prettyPrint) as String
16+
}
17+
18+
internal fun AnyRow.toJson(prettyPrint: Boolean = false): String {
19+
val jsonClass = try {
20+
Class.forName("org.jetbrains.kotlinx.dataframe.io.JsonKt")
21+
} catch (_: ClassNotFoundException) {
22+
error(
23+
"Encountered a DataRow when writing to csv/tsv/delim. This needs to be converted to JSON, so the dataframe-json dependency is required.",
24+
)
25+
}
26+
return jsonClass.getMethod("toJson", AnyRow::class.java, Boolean::class.java)
27+
.invoke(null, this, prettyPrint) as String
28+
}

‎dataframe-csv/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/writeDelim.kt

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@ import org.jetbrains.kotlinx.dataframe.documentation.DelimParams.CSV_DELIMITER
1212
import org.jetbrains.kotlinx.dataframe.documentation.DelimParams.WRITER_WRITE
1313
import org.jetbrains.kotlinx.dataframe.io.AdjustCSVFormat
1414
import org.jetbrains.kotlinx.dataframe.io.QuoteMode
15-
import org.jetbrains.kotlinx.dataframe.io.toJson
1615
import org.apache.commons.csv.QuoteMode as ApacheQuoteMode
1716

1817
/**

‎dataframe-csv/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/DelimCsvTsvTests.kt

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -796,6 +796,22 @@ class DelimCsvTsvTests {
796796
}
797797
}
798798

799+
@Test
800+
fun `json dependency test`() {
801+
val df = dataFrameOf("firstName", "lastName")(
802+
"John", "Doe",
803+
"Jane", "Doe",
804+
).group { "firstName" and "lastName" }.into { "name" }
805+
806+
df.toCsvStr(quote = '\'') shouldBe
807+
"""
808+
name
809+
'{"firstName":"John","lastName":"Doe"}'
810+
'{"firstName":"Jane","lastName":"Doe"}'
811+
812+
""".trimIndent()
813+
}
814+
799815
companion object {
800816
private val irisDataset = testCsv("irisDataset")
801817
private val simpleCsv = testCsv("testCSV")

0 commit comments

Comments
 (0)