add nulls parse option in readExcel

Author: AndreiKingsley

dataframe-excel/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/xlsx.kt

@@ -98,6 +98,8 @@ private fun setWorkbookTempDirectory() {
  * when set to false, it operates as [NameRepairStrategy.MAKE_UNIQUE],
  * ensuring unique column names will make the columns be named according to excel columns, like "A", "B", "C" etc.
  * for unstructured data.
+ * @param parseEmptyAsNull when set to true, empty strings in cells are parsed as null (default true).
+ * These cells are ignored when inferring the column’s type.
  */
 public fun DataFrame.Companion.readExcel(
     url: URL,
@@ -108,6 +110,7 @@ public fun DataFrame.Companion.readExcel(
     rowsCount: Int? = null,
     nameRepairStrategy: NameRepairStrategy = NameRepairStrategy.CHECK_UNIQUE,
     firstRowIsHeader: Boolean = true,
+    parseEmptyAsNull: Boolean = true,
 ): AnyFrame {
     setWorkbookTempDirectory()
     val wb = WorkbookFactory.create(url.openStream())
@@ -121,6 +124,7 @@ public fun DataFrame.Companion.readExcel(
             rowsCount,
             nameRepairStrategy,
             firstRowIsHeader,
+            parseEmptyAsNull
         )
     }
 }
@@ -138,6 +142,8 @@ public fun DataFrame.Companion.readExcel(
  * when set to false, it operates as [NameRepairStrategy.MAKE_UNIQUE],
  * ensuring unique column names will make the columns be named according to excel columns, like "A", "B", "C" etc.
  * for unstructured data.
+ * @param parseEmptyAsNull when set to true, empty strings in cells are parsed as null (default true).
+ * These cells are ignored when inferring the column’s type.
  */
 public fun DataFrame.Companion.readExcel(
     file: File,
@@ -148,6 +154,7 @@ public fun DataFrame.Companion.readExcel(
     rowsCount: Int? = null,
     nameRepairStrategy: NameRepairStrategy = NameRepairStrategy.CHECK_UNIQUE,
     firstRowIsHeader: Boolean = true,
+    parseEmptyAsNull: Boolean = true,
 ): AnyFrame {
     setWorkbookTempDirectory()
     @Suppress("ktlint:standard:comment-wrapping")
@@ -162,6 +169,7 @@ public fun DataFrame.Companion.readExcel(
             rowsCount,
             nameRepairStrategy,
             firstRowIsHeader,
+            parseEmptyAsNull
         )
     }
 }
@@ -179,6 +187,8 @@ public fun DataFrame.Companion.readExcel(
  * when set to false, it operates as [NameRepairStrategy.MAKE_UNIQUE],
  * ensuring unique column names will make the columns be named according to excel columns, like "A", "B", "C" etc.
  * for unstructured data.
+ * @param parseEmptyAsNull when set to true, empty strings in cells are parsed as null (default true).
+ * These cells are ignored when inferring the column’s type.
  */
 public fun DataFrame.Companion.readExcel(
     fileOrUrl: String,
@@ -189,6 +199,7 @@ public fun DataFrame.Companion.readExcel(
     rowsCount: Int? = null,
     nameRepairStrategy: NameRepairStrategy = NameRepairStrategy.CHECK_UNIQUE,
     firstRowIsHeader: Boolean = true,
+    parseEmptyAsNull: Boolean = true,
 ): AnyFrame =
     readExcel(
         asUrl(fileOrUrl),
@@ -199,6 +210,7 @@ public fun DataFrame.Companion.readExcel(
         rowsCount,
         nameRepairStrategy,
         firstRowIsHeader,
+        parseEmptyAsNull
     )
 
 /**
@@ -214,6 +226,8 @@ public fun DataFrame.Companion.readExcel(
  * when set to false, it operates as [NameRepairStrategy.MAKE_UNIQUE],
  * ensuring unique column names will make the columns be named according to excel columns, like "A", "B", "C" etc.
  * for unstructured data.
+ * @param parseEmptyAsNull when set to true, empty strings in cells are parsed as null (default true).
+ * These cells are ignored when inferring the column’s type.
  */
 public fun DataFrame.Companion.readExcel(
     inputStream: InputStream,
@@ -224,6 +238,7 @@ public fun DataFrame.Companion.readExcel(
     rowsCount: Int? = null,
     nameRepairStrategy: NameRepairStrategy = NameRepairStrategy.CHECK_UNIQUE,
     firstRowIsHeader: Boolean = true,
+    parseEmptyAsNull: Boolean = true,
 ): AnyFrame {
     setWorkbookTempDirectory()
     val wb = WorkbookFactory.create(inputStream)
@@ -237,6 +252,7 @@ public fun DataFrame.Companion.readExcel(
             rowsCount,
             nameRepairStrategy,
             firstRowIsHeader,
+            parseEmptyAsNull
         )
     }
 }
@@ -255,6 +271,8 @@ public fun DataFrame.Companion.readExcel(
  * when set to false, it operates as [NameRepairStrategy.MAKE_UNIQUE],
  * ensuring unique column names will make the columns be named according to excel columns, like "A", "B", "C" etc.
  * for unstructured data.
+ * @param parseEmptyAsNull when set to true, empty strings in cells are parsed as null (default true).
+ * These cells are ignored when inferring the column’s type.
  */
 public fun DataFrame.Companion.readExcel(
     wb: Workbook,
@@ -265,11 +283,12 @@ public fun DataFrame.Companion.readExcel(
     rowsCount: Int? = null,
     nameRepairStrategy: NameRepairStrategy = NameRepairStrategy.CHECK_UNIQUE,
     firstRowIsHeader: Boolean = true,
+    parseEmptyAsNull: Boolean = true,
 ): AnyFrame {
     val sheet: Sheet = sheetName
         ?.let { wb.getSheet(it) ?: error("Sheet with name $sheetName not found") }
         ?: wb.getSheetAt(0)
-    return readExcel(sheet, columns, formattingOptions, skipRows, rowsCount, nameRepairStrategy, firstRowIsHeader)
+    return readExcel(sheet, columns, formattingOptions, skipRows, rowsCount, nameRepairStrategy, firstRowIsHeader, parseEmptyAsNull)
 }
 
 /**
@@ -312,6 +331,7 @@ public fun DataFrame.Companion.readExcel(
     rowsCount: Int? = null,
     nameRepairStrategy: NameRepairStrategy = NameRepairStrategy.CHECK_UNIQUE,
     firstRowIsHeader: Boolean = true,
+    parseEmptyAsNull: Boolean = true,
 ): AnyFrame {
     val columnIndexes: Iterable<Int> = when {
         columns != null -> getColumnIndices(columns)
@@ -364,12 +384,18 @@ public fun DataFrame.Companion.readExcel(
         )
         columnNameCounters[nameFromCell] =
             columnNameCounters.getOrDefault(nameFromCell, 0) + 1 // increase the counter for specific column name
-        val getCellValue: (Cell?) -> Any? = when {
-            formattingOptions != null && index in formattingOptions.columnIndices -> { cell: Cell? ->
-                formattingOptions.formatter.formatCellValue(cell)
+        val getCellValue: (Cell?) -> Any? = { cell ->
+            if (cell == null) {
+                null
+            } else {
+                val rawValue: Any? = if (formattingOptions != null && index in formattingOptions.columnIndices) {
+                    formattingOptions.formatter.formatCellValue(cell)
+                } else {
+                    cell.cellValue(sheet.sheetName)
+                }
+                if (parseEmptyAsNull && rawValue is String && rawValue.isEmpty()) null
+                else rawValue
             }
-
-            else -> { cell -> cell.cellValue(sheet.sheetName) }
         }
         val values: List<Any?> = valueRowsRange.map {
             val row: Row? = sheet.getRow(it)

dataframe-excel/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/XlsxTest.kt

@@ -6,11 +6,14 @@ import kotlinx.datetime.LocalDateTime
 import org.apache.poi.ss.usermodel.WorkbookFactory
 import org.jetbrains.kotlinx.dataframe.DataFrame
 import org.jetbrains.kotlinx.dataframe.api.concat
+import org.jetbrains.kotlinx.dataframe.api.convert
 import org.jetbrains.kotlinx.dataframe.api.dataFrameOf
 import org.jetbrains.kotlinx.dataframe.api.toColumn
+import org.jetbrains.kotlinx.dataframe.api.toInt
 import org.jetbrains.kotlinx.dataframe.exceptions.DuplicateColumnNamesException
 import org.jetbrains.kotlinx.dataframe.impl.DataFrameSize
 import org.jetbrains.kotlinx.dataframe.size
+import org.jetbrains.kotlinx.dataframe.type
 import org.junit.Test
 import java.net.URL
 import java.nio.file.Files
@@ -53,6 +56,7 @@ class XlsxTest {
             "Sheet1",
             columns = "A:C",
             stringColumns = StringColumns("A:C"),
+            parseEmptyAsNull = false,
         )
         df shouldBe dataFrameOf("col1", "col2", "C")("1", "", "3")
     }
@@ -216,6 +220,7 @@ class XlsxTest {
             firstRowIsHeader = false,
             skipRows = 2,
             rowsCount = 1,
+            parseEmptyAsNull = false,
         )
 
         df shouldBe dataFrameOf(
@@ -224,4 +229,17 @@ class XlsxTest {
             "Field 3: ", "", "TEAM 1", "", "", "", "", "Staff Code:", "Staff 1", "",
         )
     }
+
+    @Test
+    fun `read columns with nulls`() {
+        val df = DataFrame.readExcel(
+            testResource("withNulls.xlsx"),
+        ).convert("age").toInt()
+        df shouldBe dataFrameOf(
+            "name" to listOf("Alice", null, "Bob"),
+            "age" to listOf(23, 27, null),
+        )
+        df["name"].type shouldBe typeOf<String?>()
+        df["age"].type shouldBe typeOf<Int?>()
+    }
 }