Improved fallback mechanism of FastDoubleParser to take into account all other locales, not just ROOT. Finished parse tests

Author: Jolanrensen

core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/convert.kt

@@ -36,8 +36,8 @@ import org.jetbrains.kotlinx.dataframe.impl.api.toLocalDateTime
 import org.jetbrains.kotlinx.dataframe.impl.api.toLocalTime
 import org.jetbrains.kotlinx.dataframe.impl.api.withRowCellImpl
 import org.jetbrains.kotlinx.dataframe.impl.headPlusArray
-import org.jetbrains.kotlinx.dataframe.io.toDataFrame
 import org.jetbrains.kotlinx.dataframe.impl.io.FastDoubleParser
+import org.jetbrains.kotlinx.dataframe.io.toDataFrame
 import java.math.BigDecimal
 import java.math.BigInteger
 import java.net.URL

core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/api/parse.kt

@@ -31,7 +31,6 @@ import org.jetbrains.kotlinx.dataframe.columns.TypeSuggestion
 import org.jetbrains.kotlinx.dataframe.columns.size
 import org.jetbrains.kotlinx.dataframe.exceptions.TypeConversionException
 import org.jetbrains.kotlinx.dataframe.hasNulls
-import org.jetbrains.kotlinx.dataframe.impl.asNullable
 import org.jetbrains.kotlinx.dataframe.impl.canParse
 import org.jetbrains.kotlinx.dataframe.impl.catchSilent
 import org.jetbrains.kotlinx.dataframe.impl.createStarProjectedType

core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/FastDoubleParser.kt

@@ -5,7 +5,6 @@ import ch.randelshofer.fastdoubleparser.NumberFormatSymbols
 import io.github.oshai.kotlinlogging.KotlinLogging
 import org.jetbrains.kotlinx.dataframe.DataFrame
 import org.jetbrains.kotlinx.dataframe.api.ParserOptions
-import org.jetbrains.kotlinx.dataframe.api.parser
 import org.jetbrains.kotlinx.dataframe.impl.api.Parsers
 import java.nio.charset.Charset
 import java.text.DecimalFormatSymbols
@@ -15,19 +14,24 @@ import java.util.Locale
 
 private val logger = KotlinLogging.logger {}
 
-// (lowercase) strings that are recognized to represent infinity and NaN in doubles in all locales
-private val INFINITIES = arrayOf("∞", "inf", "infinity", "infty")
-private val PLUS_INFINITIES = INFINITIES.map { "+$it" }
-private val MINUS_INFINITIES = INFINITIES.map { "-$it" }
-private val NANS = arrayOf("nan", "na", "n/a")
-
 /**
  * Parses a [String]/[CharSequence], [CharArray], or [ByteArray] into a [Double].
  *
  * If [ParserOptions.useFastDoubleParser] is enabled, it will try to parse the input with the
  * fast double parser library, [FastDoubleParser](https://github.com/wrandelshofer/FastDoubleParser).
  * If not, or if it fails, it will use [NumberFormat] to parse the input.
  *
+ * The [locale][locale] used by the double parser is defined like:
+ *
+ *   [parserOptions][parserOptions]`?.`[locale][ParserOptions.locale]`  ?:  `[Parsers.locale][Parsers.locale]`  :?  `[Locale.getDefault()][Locale.getDefault]
+ *
+ * [FastDoubleParser] has a fallback mechanism; In practice, this means it can recognize symbols and notations
+ * of any locale recognized by Java as long as that symbol does not conflict with the given locale.
+ *
+ * For example, if your locale uses ',' as decimal symbol, it will NOT recognize ',' as thousands separator, but it will
+ * recognize ' ', '٬', '_', ' ', etc. as thousands separator.
+ * The same holds for characters like "e", "inf", "x10", "NaN", etc.
+ *
  * Public, so it can be used in other modules.
  *
  * @param parserOptions can be supplied to configure the parser.
@@ -41,10 +45,8 @@ public class FastDoubleParser(private val parserOptions: ParserOptions? = null)
 
     private val useFastDoubleParser = parserOptions?.useFastDoubleParser ?: Parsers.useFastDoubleParser
     private val locale = parserOptions?.locale ?: Parsers.locale
-    private val fallbackLocale = Locale.ROOT
 
     private val localDecimalFormatSymbols = DecimalFormatSymbols.getInstance(locale)
-    private val fallbackDecimalFormatSymbols = DecimalFormatSymbols.getInstance(fallbackLocale)
 
     private val parser = ConfigurableDoubleParser(/* symbols = */ setupNumberFormatSymbols(), /* ignoreCase = */ true)
 
@@ -75,71 +77,69 @@ public class FastDoubleParser(private val parserOptions: ParserOptions? = null)
         }
 
         /**
-         * Builds a set with the specified char from [localDecimalFormatSymbols] and
-         * its fallback char from [fallbackDecimalFormatSymbols] if it's safe to do so.
-         * [additionals] will be added to the set too, when they're safe to add.
+         * Builds a set with the specified char from [this] and
+         * [fallbackChars] will be added to the set too, when they're safe to add.
          */
-        fun ((DecimalFormatSymbols) -> Char).fromLocalWithFallBack(vararg additionals: Char): Set<Char> =
+        fun Char.withFallback(fallbackChars: CharArray): Set<Char> =
             buildSet {
-                val getChar = this@fromLocalWithFallBack
-                val char = getChar(localDecimalFormatSymbols).lowercaseChar()
+                val char = this@withFallback.lowercaseChar()
                 add(char)
 
-                // add fallback char if it's safe to do so
-                val fallbackChar = getChar(fallbackDecimalFormatSymbols).lowercaseChar()
-                if (fallbackChar !in localChars && !localStrings.any { fallbackChar in it }) {
-                    add(fallbackChar)
-                }
-
-                // Fixes NBSP and other whitespace characters not being recognized if the user writes space instead.
-                if (char.isWhitespace()) add(' ')
+                // Treat NBSP and other whitespace characters the same.
+                if (char.isWhitespace()) addAll(WHITE_SPACES.asIterable())
 
-                // add additional chars if needed
-                for (additional in additionals) {
-                    val lowercase = additional.lowercaseChar()
+                // add fallback chars if needed
+                for (char in fallbackChars) {
+                    val lowercase = char.lowercaseChar()
                     if (lowercase !in localChars && !localStrings.any { lowercase in it }) {
                         add(lowercase)
                     }
+
+                    // Treat NBSP and other whitespace characters the same.
+                    if (char.isWhitespace()) addAll(WHITE_SPACES.asIterable())
                 }
             }
 
         /**
-         * Builds a set with the specified string from [localDecimalFormatSymbols] and
-         * its fallback string from [fallbackDecimalFormatSymbols] if it's safe to do so.
-         * [additionals] will be added to the set too, when they're safe to add.
+         * Builds a set with the specified string from [this] and
+         * [fallbackStrings] will be added to the set too, when they're safe to add.
          */
-        fun ((DecimalFormatSymbols) -> String).fromLocalWithFallBack(vararg additionals: String): Set<String> =
+        fun String.withFallback(fallbackStrings: Array<String>): Set<String> =
             buildSet {
-                val getString = this@fromLocalWithFallBack
-                val string = getString(localDecimalFormatSymbols).lowercase()
+                val string = this@withFallback.lowercase()
                 add(string)
 
-                // add fallback string if it's safe to do so
-                val fallbackString = getString(fallbackDecimalFormatSymbols).lowercase()
-                if (!fallbackString.any { it in localChars } && fallbackString !in localStrings) {
-                    add(fallbackString)
-                }
-
-                // Fixes NBSP and other whitespace characters not being recognized if the user writes space instead.
-                if (string.isBlank()) add(" ")
+                // Treat NBSP and other whitespace characters the same.
+                if (string.isBlank()) addAll(WHITE_SPACES.map { it.toString() })
 
-                // add additional strings if needed
-                for (additional in additionals) {
-                    val lowercase = additional.lowercase()
+                // add fallback strings if needed
+                for (string in fallbackStrings) {
+                    val lowercase = string.lowercase()
                     if (!lowercase.any { it in localChars } && lowercase !in localStrings) {
                         add(lowercase)
                     }
+
+                    // Treat NBSP and other whitespace characters the same.
+                    if (string.isBlank()) addAll(WHITE_SPACES.map { it.toString() })
                 }
             }
 
         return NumberFormatSymbols.fromDecimalFormatSymbols(localDecimalFormatSymbols)
-            .withPlusSign(setOf('+'))
-            .withDecimalSeparator(DecimalFormatSymbols::getDecimalSeparator.fromLocalWithFallBack())
-            .withGroupingSeparator(DecimalFormatSymbols::getGroupingSeparator.fromLocalWithFallBack())
-            .withExponentSeparator(DecimalFormatSymbols::getExponentSeparator.fromLocalWithFallBack())
-            .withMinusSign(DecimalFormatSymbols::getMinusSign.fromLocalWithFallBack())
-            .withInfinity(DecimalFormatSymbols::getInfinity.fromLocalWithFallBack(*INFINITIES))
-            .withNaN(DecimalFormatSymbols::getNaN.fromLocalWithFallBack(*NANS))
+            .withPlusSign(
+                setOf('+'),
+            ).withDecimalSeparator(
+                localDecimalFormatSymbols.decimalSeparator.withFallback(DECIMAL_SEPARATORS),
+            ).withGroupingSeparator(
+                localDecimalFormatSymbols.groupingSeparator.withFallback(GROUPING_SEPARATORS),
+            ).withExponentSeparator(
+                localDecimalFormatSymbols.exponentSeparator.withFallback(EXPONENTS),
+            ).withMinusSign(
+                localDecimalFormatSymbols.minusSign.withFallback(MINUS_SIGNS),
+            ).withInfinity(
+                localDecimalFormatSymbols.infinity.withFallback(INFINITIES),
+            ).withNaN(
+                localDecimalFormatSymbols.naN.withFallback(NANS),
+            )
     }
 
     /** Fallback method for parsing doubles. */
@@ -152,7 +152,7 @@ public class FastDoubleParser(private val parserOptions: ParserOptions? = null)
             in NANS -> Double.NaN
 
             else -> {
-                // not thread safe; must be created here
+                // NumberFormat is not thread safe; must be created in the function body
                 val numberFormat = NumberFormat.getInstance(locale)
                 val parsePosition = ParsePosition(0)
                 val result = numberFormat.parse(this, parsePosition)?.toDouble()
@@ -235,4 +235,47 @@ public class FastDoubleParser(private val parserOptions: ParserOptions? = null)
         }
         return String(chars = ca, offset = offset, length = length).parseToDoubleOrNullFallback()
     }
+
+    /**
+     * Here we store all possible decimal format symbols of all locales on the system.
+     * These will be used as fallbacks for the selected locale.
+     * They are only added by [withFallback] if they don't interfere with symbols already in the provided [locale]
+     * (so ',' is not added as grouping separator if '.' is already the locale's decimal separator).
+     */
+    internal companion object {
+        private val allDecimalFormatSymbols by lazy {
+            Locale.getAvailableLocales().map { DecimalFormatSymbols.getInstance(it) }
+        }
+        val MINUS_SIGNS by lazy {
+            allDecimalFormatSymbols.mapNotNullTo(mutableSetOf()) { it.minusSign }.toCharArray()
+        }
+
+        // (lowercase) strings that are recognized to represent infinity and NaN in doubles in all locales
+        val INFINITIES by lazy {
+            allDecimalFormatSymbols.mapNotNullTo(mutableSetOf()) { it.infinity }
+                .plus(arrayOf("∞", "inf", "infinity", "infty"))
+                .toTypedArray()
+        }
+        val PLUS_INFINITIES by lazy { INFINITIES.map { "+$it" }.toTypedArray() }
+        val MINUS_INFINITIES by lazy {
+            INFINITIES.flatMap { inf -> MINUS_SIGNS.map { min -> min + inf } }.toTypedArray()
+        }
+        val NANS by lazy {
+            allDecimalFormatSymbols.mapNotNullTo(mutableSetOf()) { it.naN }
+                .plus(arrayOf("nan", "na", "n/a"))
+                .toTypedArray()
+        }
+        val GROUPING_SEPARATORS by lazy {
+            allDecimalFormatSymbols.mapNotNullTo(mutableSetOf()) { it.groupingSeparator }.toCharArray()
+        }
+        val DECIMAL_SEPARATORS by lazy {
+            allDecimalFormatSymbols.flatMapTo(mutableSetOf()) {
+                listOfNotNull(it.decimalSeparator, it.monetaryDecimalSeparator)
+            }.toCharArray()
+        }
+        val EXPONENTS by lazy {
+            allDecimalFormatSymbols.mapNotNullTo(mutableSetOf()) { it.exponentSeparator }.toTypedArray()
+        }
+        val WHITE_SPACES = charArrayOf(' ', '\u00A0', '\u2009', '\u202F', '\t')
+    }
 }