Merge branch 'master' into continued-columnsselectiondsl

Author: Jolanrensen

.space.kts

@@ -20,8 +20,21 @@ Kotlin Dataframe aims to reconcile Kotlin's static typing with the dynamic natur
 
 Integrates with [Kotlin kernel for Jupyter](https://github.com/Kotlin/kotlin-jupyter). Inspired by [krangl](https://github.com/holgerbrandl/krangl), Kotlin Collections and [pandas](https://pandas.pydata.org/)
 
+## Documentation
+
 Explore [**documentation**](https://kotlin.github.io/dataframe/overview.html) for details.
 
+You could find the following articles there:
+
+* [Get started with Kotlin DataFrame](https://kotlin.github.io/dataframe/gettingstarted.html)
+* [Working with Data Schemas](https://kotlin.github.io/dataframe/schemas.html)
+* [Full list of all supported operations](https://kotlin.github.io/dataframe/operations.html)
+    * [Reading from SQL databases](https://kotlin.github.io/dataframe/readsqldatabases.html)
+    * [Reading/writing from/to different file formats like JSON, CSV, Apache Arrow](https://kotlin.github.io/dataframe/read.html)
+    * [Joining a few dataframes](https://kotlin.github.io/dataframe/join.html)
+    * [GroupBy operation](https://kotlin.github.io/dataframe/groupby.html)
+* [Rendering to HTML](https://kotlin.github.io/dataframe/tohtml.html#jupyter-notebooks)
+
 ## Setup
 
 ### Gradle for JVM
@@ -180,19 +193,6 @@ or specific version:
   * `ColumnGroup` — contains columns
   * `FrameColumn` — contains dataframes
 
-
-## Kotlin, Kotlin Jupyter, OpenAPI, Arrow and JDK versions
-
-This table shows the mapping between main library component versions and minimum supported Java versions.
-
-| Kotlin DataFrame Version | Minimum Java Version | Kotlin Version | Kotlin Jupyter Version | OpenAPI version | Apache Arrow version |
-|--------------------------|----------------------|----------------|------------------------|-----------------|----------------------|
-| 0.10.0                   | 8                    | 1.8.20         | 0.11.0-358             | 3.0.0           | 11.0.0               |
-| 0.10.1                   | 8                    | 1.8.20         | 0.11.0-358             | 3.0.0           | 11.0.0               |
-| 0.11.0                   | 8                    | 1.8.20         | 0.11.0-358             | 3.0.0           | 11.0.0               |
-| 0.11.1                   | 8                    | 1.8.20         | 0.11.0-358             | 3.0.0           | 11.0.0               |
-| 0.12.0                   | 8                    | 1.9.0          | 0.11.0-358             | 3.0.0           | 11.0.0               |
-
 ## Usage example
 
 **Create:**
@@ -271,6 +271,20 @@ clean
 
 [Try it in **Datalore**](https://datalore.jetbrains.com/view/notebook/vq5j45KWkYiSQnACA2Ymij) and explore [**more examples here**](examples).
 
+## Kotlin, Kotlin Jupyter, OpenAPI, Arrow and JDK versions
+
+This table shows the mapping between main library component versions and minimum supported Java versions.
+
+| Kotlin DataFrame Version | Minimum Java Version | Kotlin Version | Kotlin Jupyter Version | OpenAPI version | Apache Arrow version |
+|--------------------------|----------------------|----------------|------------------------|-----------------|----------------------|
+| 0.10.0                   | 8                    | 1.8.20         | 0.11.0-358             | 3.0.0           | 11.0.0               |
+| 0.10.1                   | 8                    | 1.8.20         | 0.11.0-358             | 3.0.0           | 11.0.0               |
+| 0.11.0                   | 8                    | 1.8.20         | 0.11.0-358             | 3.0.0           | 11.0.0               |
+| 0.11.1                   | 8                    | 1.8.20         | 0.11.0-358             | 3.0.0           | 11.0.0               |
+| 0.12.0                   | 8                    | 1.9.0          | 0.11.0-358             | 3.0.0           | 11.0.0               |
+| 0.12.1                   | 8                    | 1.9.0          | 0.11.0-358             | 3.0.0           | 11.0.0               |
+
+
 ## Code of Conduct
 
 This project and the corresponding community are governed by the [JetBrains Open Source and Community Code of Conduct](https://confluence.jetbrains.com/display/ALL/JetBrains+Open+Source+and+Community+Code+of+Conduct). Please make sure you read it.

README.md

@@ -33,4 +33,4 @@
     - `Level.WARNING` messages are changed to `Level.ERROR`
     - `Level.ERROR` messages and their functions are removed.
     - Update regions in the file accordingly.
-20. Update Notebook examples, both in the project as well as on Datalore.
+20. Update Notebook examples, both in the project and on Datalore.

RELEASE_CHECK_LIST.md

@@ -333,7 +333,7 @@ public fun NullabilityOptions.applyNullability(data: List<Any?>, expectedNulls:
 
 public inline fun <reified T> Iterable<T>.toColumn(
     name: String = "",
-    infer: Infer = Infer.None,
+    infer: Infer = Infer.Nulls,
 ): DataColumn<T> =
     (
         if (infer == Infer.Type) DataColumn.createWithTypeInference(name, asList())

core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/TypeConversions.kt

@@ -22,14 +22,14 @@ internal fun <T, C> RenameClause<T, C>.renameImpl(newNames: Array<out String>):
 internal fun <T, C> RenameClause<T, C>.renameImpl(transform: (ColumnWithPath<C>) -> String): DataFrame<T> {
     // get all selected columns and their paths
     val selectedColumnsWithPath = df.getColumnsWithPaths(columns)
-        .associateBy { it.data }
+        .associateBy { it.path }
     // gather a tree of all columns where the nodes will be renamed
     val tree = df.getColumnsWithPaths { colsAtAnyDepth() }.collectTree()
 
     // perform rename in nodes
-    tree.allChildrenNotNull().forEach { node ->
+    tree.allChildrenNotNull().map { it to it.pathFromRoot() }.forEach { (node, originalPath) ->
         // Check if the current node/column is a selected column and, if so, get its ColumnWithPath
-        val column = selectedColumnsWithPath[node.data] ?: return@forEach
+        val column = selectedColumnsWithPath[originalPath] ?: return@forEach
         // Use the found selected ColumnWithPath to query for the new name
         val newColumnName = transform(column)
         node.name = newColumnName

core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/api/rename.kt

@@ -21,13 +21,16 @@ import org.jetbrains.kotlinx.dataframe.impl.getListType
 import org.jetbrains.kotlinx.dataframe.impl.projectUpTo
 import org.jetbrains.kotlinx.dataframe.impl.schema.getPropertiesOrder
 import java.lang.reflect.InvocationTargetException
+import java.lang.reflect.Method
 import java.time.temporal.Temporal
 import kotlin.reflect.KClass
 import kotlin.reflect.KProperty
 import kotlin.reflect.KVisibility
 import kotlin.reflect.full.isSubclassOf
 import kotlin.reflect.full.memberProperties
+import kotlin.reflect.full.primaryConstructor
 import kotlin.reflect.full.withNullability
+import kotlin.reflect.jvm.isAccessible
 import kotlin.reflect.jvm.javaField
 import kotlin.reflect.typeOf
 
@@ -141,7 +144,26 @@ internal fun convertToDataFrame(
         val property = it
         if (excludes.contains(property)) return@mapNotNull null
 
+        class ValueClassConverter(val unbox: Method, val box: Method)
+
+        val valueClassConverter = (it.returnType.classifier as? KClass<*>)?.let { kClass ->
+            if (!kClass.isValue) null else {
+                val constructor =
+                    requireNotNull(kClass.primaryConstructor) { "value class $kClass is expected to have primary constructor, but couldn't obtain it" }
+                val parameter = constructor.parameters.singleOrNull()
+                    ?: error("conversion of value class $kClass with multiple parameters in constructor is not yet supported")
+                // there's no need to unwrap if underlying field is nullable
+                if (parameter.type.isMarkedNullable) return@let null
+                // box and unbox impl methods are part of binary API of value classes
+                // https://youtrack.jetbrains.com/issue/KT-50518/Boxing-Unboxing-methods-for-JvmInline-value-classes-should-be-public-accessible
+                val unbox = kClass.java.getMethod("unbox-impl")
+                val box = kClass.java.methods.single { it.name == "box-impl" }
+                val valueClassConverter = ValueClassConverter(unbox, box)
+                valueClassConverter
+            }
+        }
         property.javaField?.isAccessible = true
+        property.isAccessible = true
 
         var nullable = false
         var hasExceptions = false
@@ -151,7 +173,19 @@ internal fun convertToDataFrame(
                 null
             } else {
                 val value = try {
-                    it.call(obj)
+                    val value = it.call(obj)
+                    /**
+                     * here we do what compiler does
+                     * @see org.jetbrains.kotlinx.dataframe.api.CreateDataFrameTests.testKPropertyGetLibrary
+                     */
+                    if (valueClassConverter != null) {
+                        val var1 = value?.let {
+                            valueClassConverter.unbox.invoke(it)
+                        }
+                        var1?.let { valueClassConverter.box.invoke(null, var1) }
+                    } else {
+                        value
+                    }
                 } catch (e: InvocationTargetException) {
                     hasExceptions = true
                     e.targetException