Parse String column to nullable Double with correct locale (#226)

* Test on parsing Double values inside ArrowWriter

* Fix stringColumn.convertTo<Double> with cross-nullable combinations

Author: Kopilov

core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/TypeConversions.kt

@@ -95,6 +95,14 @@ public fun <T> DataColumn<T>.castToNullable(): DataColumn<T?> = cast()
 
 public fun <T> ColumnReference<T>.castToNullable(): ColumnReference<T?> = cast()
 
+public fun AnyCol.setNullable(nullable: Boolean): AnyCol {
+    return if (nullable) {
+        this.castToNullable()
+    } else {
+        this.castToNotNullable()
+    }
+}
+
 // region to array
 
 public inline fun <reified T> DataColumn<T>.toTypedArray(): Array<T> = toList().toTypedArray()

core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/convert.kt

@@ -36,6 +36,8 @@ import java.time.LocalTime
 import java.util.*
 import kotlin.reflect.KProperty
 import kotlin.reflect.KType
+import kotlin.reflect.full.isSubtypeOf
+import kotlin.reflect.full.withNullability
 import kotlin.reflect.typeOf
 
 public fun <T, C> DataFrame<T>.convert(columns: ColumnsSelector<T, C>): Convert<T, C> =
@@ -103,8 +105,9 @@ public fun <T, C> Convert<T, C>.to(columnConverter: DataFrame<T>.(DataColumn<C>)
 
 public inline fun <reified C> AnyCol.convertTo(): DataColumn<C> = convertTo(typeOf<C>()) as DataColumn<C>
 public fun AnyCol.convertTo(newType: KType): AnyCol {
-    if (this.type() == typeOf<String>() && newType == typeOf<Double>()) return (this as DataColumn<String>).convertToDouble()
-    if (this.type() == typeOf<String?>() && newType == typeOf<Double?>()) return (this as DataColumn<String?>).convertToDouble()
+    if (this.type().withNullability(true).isSubtypeOf(typeOf<String?>()) && newType.withNullability(true) == typeOf<Double?>()) {
+        return (this as DataColumn<String?>).convertToDouble().setNullable(newType.isMarkedNullable)
+    }
     return convertToTypeImpl(newType)
 }
 

core/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/ParserTests.kt

@@ -132,14 +132,14 @@ class ParserTests {
     fun `converting String to Double in different locales`() {
         val currentLocale = Locale.getDefault()
         try {
-            // Test 36 behaviour combinations:
+            // Test 45 behaviour combinations:
 
             // 3 source columns
             val columnDot = columnOf("12.345", "67.890")
             val columnComma = columnOf("12,345", "67,890")
             val columnMixed = columnOf("12.345", "67,890")
             // *
-            // (3 locales as converting parameter + original converting)
+            // (3 locales as converting parameter + original converting + original converting to nullable)
             val parsingLocaleNotDefined: Locale? = null
             val parsingLocaleUsesDot: Locale = Locale.forLanguageTag("en-US")
             val parsingLocaleUsesComma: Locale = Locale.forLanguageTag("ru-RU")
@@ -152,6 +152,10 @@ class ParserTests {
             columnComma.convertTo<Double>().shouldBe(columnOf(12345.0, 67890.0))
             columnMixed.convertTo<Double>().shouldBe(columnOf(12.345, 67890.0))
 
+            columnDot.convertTo<Double?>().shouldBe(columnOf(12.345, 67.89))
+            columnComma.convertTo<Double?>().shouldBe(columnOf(12345.0, 67890.0))
+            columnMixed.convertTo<Double?>().shouldBe(columnOf(12.345, 67890.0))
+
             columnDot.convertToDouble(parsingLocaleNotDefined).shouldBe(columnOf(12.345, 67.89))
             columnComma.convertToDouble(parsingLocaleNotDefined).shouldBe(columnOf(12345.0, 67890.0))
             columnMixed.convertToDouble(parsingLocaleNotDefined).shouldBe(columnOf(12.345, 67890.0))
@@ -170,6 +174,10 @@ class ParserTests {
             columnComma.convertTo<Double>().shouldBe(columnOf(12345.0, 67890.0))
             columnMixed.convertTo<Double>().shouldBe(columnOf(12.345, 67890.0))
 
+            columnDot.convertTo<Double?>().shouldBe(columnOf(12.345, 67.89))
+            columnComma.convertTo<Double?>().shouldBe(columnOf(12345.0, 67890.0))
+            columnMixed.convertTo<Double?>().shouldBe(columnOf(12.345, 67890.0))
+
             columnDot.convertToDouble(parsingLocaleNotDefined).shouldBe(columnOf(12.345, 67.89))
             columnComma.convertToDouble(parsingLocaleNotDefined).shouldBe(columnOf(12345.0, 67890.0))
             columnMixed.convertToDouble(parsingLocaleNotDefined).shouldBe(columnOf(12.345, 67890.0))
@@ -188,6 +196,10 @@ class ParserTests {
             columnComma.convertTo<Double>().shouldBe(columnOf(12.345, 67.89))
             columnMixed.convertTo<Double>().shouldBe(columnOf(12.345, 67890.0))
 
+            columnDot.convertTo<Double?>().shouldBe(columnOf(12.345, 67.89))
+            columnComma.convertTo<Double?>().shouldBe(columnOf(12.345, 67.89))
+            columnMixed.convertTo<Double?>().shouldBe(columnOf(12.345, 67890.0))
+
             columnDot.convertToDouble(parsingLocaleNotDefined).shouldBe(columnOf(12.345, 67.89))
             columnComma.convertToDouble(parsingLocaleNotDefined).shouldBe(columnOf(12.345, 67.89))
             columnMixed.convertToDouble(parsingLocaleNotDefined).shouldBe(columnOf(12.345, 67890.0))

dataframe-arrow/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/ArrowKtTest.kt

@@ -3,6 +3,10 @@ package org.jetbrains.kotlinx.dataframe.io
 import io.kotest.assertions.throwables.shouldThrow
 import io.kotest.matchers.collections.shouldContain
 import io.kotest.matchers.shouldBe
+import org.apache.arrow.vector.types.FloatingPointPrecision
+import org.apache.arrow.vector.types.pojo.ArrowType
+import org.apache.arrow.vector.types.pojo.Field
+import org.apache.arrow.vector.types.pojo.FieldType
 import org.apache.arrow.vector.types.pojo.Schema
 import org.apache.arrow.vector.util.Text
 import org.jetbrains.kotlinx.dataframe.DataColumn
@@ -23,6 +27,7 @@ import java.io.File
 import java.net.URL
 import java.time.LocalDate
 import java.time.LocalDateTime
+import java.util.Locale
 import kotlin.reflect.typeOf
 
 internal class ArrowKtTest {
@@ -238,4 +243,27 @@ internal class ArrowKtTest {
         DataFrame.readArrowFeather(testLoyalNullable)["settled"].type() shouldBe typeOf<LocalDateTime?>()
         DataFrame.readArrowFeather(testLoyalNullable)["settled"].values() shouldBe arrayOfNulls<LocalDate>(frameRenaming.rowsCount()).asList()
     }
+
+    @Test
+    fun testParsing() {
+        val columnStringDot = columnOf("12.345", "67.890")
+        val columnStringComma = columnOf("12,345", "67,890")
+        val frameString = dataFrameOf("columnDot", "columnComma")(columnStringDot, columnStringComma)
+        val columnDoubleFraction = columnOf(12.345, 67.890)
+        val columnDoubleRound = columnOf(12345.0, 67890.0)
+        val targetType = FieldType.notNullable(ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE))
+        val targetSchema = Schema(listOf(Field("columnDot", targetType, emptyList()), Field("columnComma", targetType, emptyList())))
+
+        val currentLocale = Locale.getDefault()
+        try {
+            Locale.setDefault(Locale.forLanguageTag("en-US"))
+            val serializedAsUs = frameString.arrowWriter(targetSchema).saveArrowFeatherToByteArray()
+            DataFrame.readArrowFeather(serializedAsUs) shouldBe dataFrameOf("columnDot", "columnComma")(columnDoubleFraction, columnDoubleRound)
+            Locale.setDefault(Locale.forLanguageTag("ru-RU"))
+            val serializedAsRu = frameString.arrowWriter(targetSchema).saveArrowFeatherToByteArray()
+            DataFrame.readArrowFeather(serializedAsRu) shouldBe dataFrameOf("columnDot", "columnComma")(columnDoubleFraction, columnDoubleFraction)
+        } finally {
+            Locale.setDefault(currentLocale)
+        }
+    }
 }