Convert StructVector into ColumnGroup instead of DataColumn<Map<String, Any?>>

Author: koperagen

dataframe-arrow/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/arrowReadingImpl.kt

@@ -19,6 +19,7 @@ import org.apache.arrow.vector.DateMilliVector
 import org.apache.arrow.vector.Decimal256Vector
 import org.apache.arrow.vector.DecimalVector
 import org.apache.arrow.vector.DurationVector
+import org.apache.arrow.vector.FieldVector
 import org.apache.arrow.vector.Float4Vector
 import org.apache.arrow.vector.Float8Vector
 import org.apache.arrow.vector.IntVector
@@ -293,10 +294,16 @@ private fun List<Nothing?>.withTypeNullable(
     return this to nothingType(nullable)
 }
 
-private fun readField(root: VectorSchemaRoot, field: Field, nullability: NullabilityOptions): AnyBaseCol {
+private fun readField(vector: FieldVector, field: Field, nullability: NullabilityOptions): AnyBaseCol {
     try {
-        val range = 0 until root.rowCount
-        val (list, type) = when (val vector = root.getVector(field)) {
+        val range = 0 until vector.valueCount
+        if (vector is StructVector) {
+            val columns = field.children.map { childField ->
+                readField(vector.getChild(childField.name), childField, nullability)
+            }
+            return DataColumn.createColumnGroup(field.name, columns.toDataFrame())
+        }
+        val (list, type) = when (vector) {
             is VarCharVector -> vector.values(range).withTypeNullable(field.isNullable, nullability)
 
             is LargeVarCharVector -> vector.values(range).withTypeNullable(field.isNullable, nullability)
@@ -357,8 +364,6 @@ private fun readField(root: VectorSchemaRoot, field: Field, nullability: Nullabi
 
             is TimeStampSecVector -> vector.values(range).withTypeNullable(field.isNullable, nullability)
 
-            is StructVector -> vector.values(range).withTypeNullable(field.isNullable, nullability)
-
             is NullVector -> vector.values(range).withTypeNullable(field.isNullable, nullability)
 
             else -> {
@@ -371,6 +376,9 @@ private fun readField(root: VectorSchemaRoot, field: Field, nullability: Nullabi
     }
 }
 
+private fun readField(root: VectorSchemaRoot, field: Field, nullability: NullabilityOptions): AnyBaseCol =
+    readField(root.getVector(field), field, nullability)
+
 /**
  * Read [Arrow interprocess streaming format](https://arrow.apache.org/docs/java/ipc.html#writing-and-reading-streaming-format) data from existing [channel]
  */

dataframe-arrow/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/ArrowKtTest.kt

@@ -25,7 +25,6 @@ import org.apache.arrow.vector.types.pojo.Field
 import org.apache.arrow.vector.types.pojo.FieldType
 import org.apache.arrow.vector.types.pojo.Schema
 import org.apache.arrow.vector.util.ByteArrayReadableSeekableByteChannel
-import org.apache.arrow.vector.util.Text
 import org.duckdb.DuckDBConnection
 import org.duckdb.DuckDBResultSet
 import org.jetbrains.kotlinx.dataframe.AnyFrame
@@ -39,7 +38,6 @@ import org.jetbrains.kotlinx.dataframe.api.dataFrameOf
 import org.jetbrains.kotlinx.dataframe.api.map
 import org.jetbrains.kotlinx.dataframe.api.pathOf
 import org.jetbrains.kotlinx.dataframe.api.remove
-import org.jetbrains.kotlinx.dataframe.api.toColumn
 import org.jetbrains.kotlinx.dataframe.exceptions.TypeConverterNotFoundException
 import org.junit.Assert
 import org.junit.Test
@@ -68,13 +66,11 @@ internal class ArrowKtTest {
         val df = DataFrame.readArrowFeather(feather)
         val a by columnOf("one")
         val b by columnOf(2.0)
-        val c by listOf(
-            mapOf(
-                "c1" to Text("inner"),
-                "c2" to 4.0,
-                "c3" to 50.0,
-            ) as Map<String, Any?>,
-        ).toColumn()
+        val c by columnOf(
+            "c1" to columnOf("inner"),
+            "c2" to columnOf(4.0),
+            "c3" to columnOf(50.0),
+        )
         val d by columnOf("four")
         val expected = dataFrameOf(a, b, c, d)
         df shouldBe expected