added double parsing test for all locales and streamlined behavior for "-" in RTL languages with NumberFormat

Author: Jolanrensen

core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/FastDoubleParser.kt

@@ -43,6 +43,10 @@ public class FastDoubleParser(private val parserOptions: ParserOptions) {
 
     private val parser = ConfigurableDoubleParser(/* symbols = */ setupNumberFormatSymbols(), /* ignoreCase = */ true)
 
+    // Fix for Java 8 RTL languages minus sign not being recognized
+    private val minusSignIsFormatSymbol =
+        Character.getType(localDecimalFormatSymbols.minusSign) == Character.FORMAT.toInt()
+
     /**
      * Sets up the [NumberFormatSymbols] for the [ConfigurableDoubleParser] based on
      * [localDecimalFormatSymbols] with fallbacks from [fallbackDecimalFormatSymbols].
@@ -177,6 +181,17 @@ public class FastDoubleParser(private val parserOptions: ParserOptions) {
     ): Double? {
         if (parserOptions.useFastDoubleParser && charset in supportedFastCharsets) {
             try {
+                // Fixes RTL minus sign not being recognized
+                if (minusSignIsFormatSymbol && ba.toString(charset).startsWith(localDecimalFormatSymbols.minusSign)) {
+                    val localMinusSize = localDecimalFormatSymbols.minusSign.toString().toByteArray(charset).size
+                    val fallbackMinusSize = fallbackDecimalFormatSymbols.minusSign.toString().toByteArray(charset).size
+                    val newOffset = (localMinusSize - fallbackMinusSize).coerceAtLeast(0)
+                    val newBa = ba.copyOf()
+                    fallbackDecimalFormatSymbols.minusSign.toString().toByteArray(charset)
+                        .copyInto(destination = newBa, destinationOffset = newOffset)
+
+                    return parser.parseDouble(newBa, newOffset, length - newOffset)
+                }
                 return parser.parseDouble(ba, offset, length)
             } catch (e: Exception) {
                 logger.debug(e) {
@@ -199,6 +214,15 @@ public class FastDoubleParser(private val parserOptions: ParserOptions) {
     public fun parseOrNull(cs: CharSequence): Double? {
         if (parserOptions.useFastDoubleParser) {
             try {
+                // Fixes RTL minus sign not being recognized
+                if (minusSignIsFormatSymbol && cs.startsWith(localDecimalFormatSymbols.minusSign)) {
+                    val newCs = cs.toString().replaceFirst(
+                        localDecimalFormatSymbols.minusSign,
+                        fallbackDecimalFormatSymbols.minusSign,
+                    )
+                    return parser.parseDouble(newCs)
+                }
+
                 return parser.parseDouble(cs)
             } catch (e: Exception) {
                 logger.debug(e) {
@@ -219,6 +243,12 @@ public class FastDoubleParser(private val parserOptions: ParserOptions) {
     public fun parseOrNull(ca: CharArray, offset: Int = 0, length: Int = ca.size): Double? {
         if (parserOptions.useFastDoubleParser) {
             try {
+                // Fixes RTL minus sign not being recognized.
+                if (minusSignIsFormatSymbol && ca.firstOrNull() == localDecimalFormatSymbols.minusSign) {
+                    val newCa = ca.copyOf()
+                    newCa[0] = fallbackDecimalFormatSymbols.minusSign
+                    return parser.parseDouble(newCa, offset, length)
+                }
                 return parser.parseDouble(ca, offset, length)
             } catch (e: Exception) {
                 logger.debug(e) {

core/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/FastDoubleParserTests.kt

@@ -6,6 +6,7 @@ import org.jetbrains.kotlinx.dataframe.impl.io.FastDoubleParser
 import org.junit.After
 import org.junit.Before
 import org.junit.Test
+import java.text.NumberFormat
 import java.util.Locale
 
 private const val LOG_LEVEL = "org.slf4j.simpleLogger.defaultLogLevel"
@@ -160,4 +161,45 @@ class FastDoubleParserTests {
         // ByteArray
         numbers.map { parser.parseOrNull(it.toByteArray()) }.shouldContainInOrder(expectedDoubles)
     }
+
+    @Test
+    fun `fast parse any locale`() {
+        val locales = Locale.getAvailableLocales()
+        val doubles = listOf(
+            12.45,
+            -12.45,
+            100_123.35,
+            -204_235.23,
+            1.234e3,
+            -345.122,
+            0.0,
+            Double.POSITIVE_INFINITY,
+            Double.NEGATIVE_INFINITY,
+            Double.NaN,
+        )
+
+        for (locale in locales) {
+            val parser = FastDoubleParser(ParserOptions(locale = locale, useFastDoubleParser = true))
+            val formatter = NumberFormat.getInstance(locale)
+            for (double in doubles) {
+                val formatted = formatter.format(double)
+                val parsedByNumberFormatter = formatter.parse(formatted)?.toDouble()
+
+                val parsedString = parser.parseOrNull(formatted)
+                assert(double == parsedString || double.isNaN() && parsedString?.isNaN() == true) {
+                    "Failed to parse $formatted with locale $locale. Expected $double, got $parsedString. NumberFormat parsed it like: $parsedByNumberFormatter"
+                }
+
+                val parsedCharArray = parser.parseOrNull(formatted.toCharArray())
+                assert(double == parsedCharArray || double.isNaN() && parsedCharArray?.isNaN() == true) {
+                    "Failed to parse $formatted with locale $locale. Expected $double, got $parsedCharArray. NumberFormat parsed it like: $parsedByNumberFormatter"
+                }
+
+                val parsedByteArray = parser.parseOrNull(formatted.toByteArray())
+                assert(double == parsedByteArray || double.isNaN() && parsedByteArray?.isNaN() == true) {
+                    "Failed to parse $formatted with locale $locale. Expected $double, got $parsedByteArray. NumberFormat parsed it like: $parsedByNumberFormatter"
+                }
+            }
+        }
+    }
 }