Merge branch 'refs/heads/master' into column-selector-docs3

Author: Jolanrensen

README.md

@@ -1,4 +1,4 @@
-# Kotlin Dataframe: typesafe in-memory structured data processing for JVM
+# Kotlin DataFrame: typesafe in-memory structured data processing for JVM
 [![JetBrains incubator project](https://jb.gg/badges/incubator.svg)](https://confluence.jetbrains.com/display/ALL/JetBrains+on+GitHub)
 [![Kotlin component alpha stability](https://img.shields.io/badge/project-alpha-kotlin.svg?colorA=555555&colorB=DB3683&label=&logo=kotlin&logoColor=ffffff&logoWidth=10)](https://kotlinlang.org/docs/components-stability.html)
 [![Kotlin](https://img.shields.io/badge/kotlin-1.9.22-blue.svg?logo=kotlin)](http://kotlinlang.org)
@@ -8,7 +8,7 @@
 [![GitHub License](https://img.shields.io/badge/license-Apache%20License%202.0-blue.svg?style=flat)](http://www.apache.org/licenses/LICENSE-2.0)
 [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/Kotlin/dataframe/HEAD)
 
-Kotlin Dataframe aims to reconcile Kotlin's static typing with the dynamic nature of data by utilizing both the full power of the Kotlin language and the opportunities provided by intermittent code execution in Jupyter notebooks and REPL.   
+Kotlin DataFrame aims to reconcile Kotlin's static typing with the dynamic nature of data by utilizing both the full power of the Kotlin language and the opportunities provided by intermittent code execution in Jupyter notebooks and REPL.   
 
 * **Hierarchical** — represents hierarchical data structures, such as JSON or a tree of JVM objects.
 * **Functional** — data processing pipeline is organized in a chain of `DataFrame` transformation operations. Every operation returns a new instance of `DataFrame` reusing underlying storage wherever it's possible.
@@ -18,7 +18,7 @@ Kotlin Dataframe aims to reconcile Kotlin's static typing with the dynamic natur
 * **Interoperable** — convertable with Kotlin data classes and collections.
 * **Generic** — can store objects of any type, not only numbers or strings.
 * **Typesafe** — on-the-fly generation of extension properties for type safe data access with Kotlin-style care for null safety.
-* **Polymorphic** — type compatibility derives from column schema compatibility. You can define a function that requires a special subset of columns in dataframe but doesn't care about other columns.
+* **Polymorphic** — type compatibility derives from column schema compatibility. You can define a function that requires a special subset of columns in a dataframe but doesn't care about other columns.
 
 Integrates with [Kotlin kernel for Jupyter](https://github.com/Kotlin/kotlin-jupyter). Inspired by [krangl](https://github.com/holgerbrandl/krangl), Kotlin Collections and [pandas](https://pandas.pydata.org/)
 
@@ -33,7 +33,7 @@ You could find the following articles there:
 * [Full list of all supported operations](https://kotlin.github.io/dataframe/operations.html)
     * [Reading from SQL databases](https://kotlin.github.io/dataframe/readsqldatabases.html)
     * [Reading/writing from/to different file formats like JSON, CSV, Apache Arrow](https://kotlin.github.io/dataframe/read.html)
-    * [Joining a few dataframes](https://kotlin.github.io/dataframe/join.html)
+    * [Joining dataframes](https://kotlin.github.io/dataframe/join.html)
     * [GroupBy operation](https://kotlin.github.io/dataframe/groupby.html)
 * [Rendering to HTML](https://kotlin.github.io/dataframe/tohtml.html#jupyter-notebooks)
 
@@ -99,13 +99,13 @@ fun main() {
 
 ## Getting started in Jupyter Notebook / Kotlin Notebook
 
-Install [Kotlin kernel](https://github.com/Kotlin/kotlin-jupyter) for [Jupyter](https://jupyter.org/)
+Install the [Kotlin kernel](https://github.com/Kotlin/kotlin-jupyter) for [Jupyter](https://jupyter.org/)
 
-Import stable `dataframe` version into notebook: 
+Import the stable `dataframe` version into a notebook: 
 ```
 %use dataframe
 ```
-or specific version:
+or a specific version:
 ```
 %use dataframe(<version>)
 ```
@@ -205,7 +205,7 @@ clean
     }
 ```
 
-Check it out on [**Datalore**](https://datalore.jetbrains.com/view/notebook/vq5j45KWkYiSQnACA2Ymij) to get a better visual impression of what happens and what the hierarchical DataFrame structure looks like. 
+Check it out on [**Datalore**](https://datalore.jetbrains.com/view/notebook/vq5j45KWkYiSQnACA2Ymij) to get a better visual impression of what happens and what the hierarchical dataframe structure looks like. 
 
 Explore [**more examples here**](examples).
 
@@ -229,4 +229,4 @@ This project and the corresponding community are governed by the [JetBrains Open
 
 ## License
 
-Kotlin Dataframe is licensed under the [Apache 2.0 License](LICENSE).
+Kotlin DataFrame is licensed under the [Apache 2.0 License](LICENSE).

core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/DataColumnType.kt

@@ -2,10 +2,13 @@ package org.jetbrains.kotlinx.dataframe.api
 
 import org.jetbrains.kotlinx.dataframe.AnyCol
 import org.jetbrains.kotlinx.dataframe.columns.ColumnKind
+import org.jetbrains.kotlinx.dataframe.impl.isNothing
+import org.jetbrains.kotlinx.dataframe.impl.projectTo
 import org.jetbrains.kotlinx.dataframe.type
 import org.jetbrains.kotlinx.dataframe.typeClass
 import kotlin.reflect.KClass
 import kotlin.reflect.KType
+import kotlin.reflect.KTypeProjection
 import kotlin.reflect.full.isSubclassOf
 import kotlin.reflect.full.isSubtypeOf
 import kotlin.reflect.typeOf
@@ -16,7 +19,8 @@ public fun AnyCol.isFrameColumn(): Boolean = kind() == ColumnKind.Frame
 
 public fun AnyCol.isValueColumn(): Boolean = kind() == ColumnKind.Value
 
-public fun AnyCol.isSubtypeOf(type: KType): Boolean = this.type.isSubtypeOf(type) && (!this.type.isMarkedNullable || type.isMarkedNullable)
+public fun AnyCol.isSubtypeOf(type: KType): Boolean =
+    this.type.isSubtypeOf(type) && (!this.type.isMarkedNullable || type.isMarkedNullable)
 
 public inline fun <reified T> AnyCol.isSubtypeOf(): Boolean = isSubtypeOf(typeOf<T>())
 
@@ -26,9 +30,23 @@ public fun AnyCol.isNumber(): Boolean = isSubtypeOf<Number?>()
 
 public fun AnyCol.isList(): Boolean = typeClass == List::class
 
-public fun AnyCol.isComparable(): Boolean = isSubtypeOf<Comparable<*>?>()
+/**
+ * Returns `true` if [this] column is comparable, i.e. its type is a subtype of [Comparable] and its
+ * type argument is not [Nothing].
+ */
+public fun AnyCol.isComparable(): Boolean =
+    isSubtypeOf<Comparable<*>?>() &&
+        type().projectTo(Comparable::class).arguments[0].let {
+            it != KTypeProjection.STAR &&
+                it.type?.isNothing != true
+        }
 
 @PublishedApi
 internal fun AnyCol.isPrimitive(): Boolean = typeClass.isPrimitive()
 
-internal fun KClass<*>.isPrimitive(): Boolean = isSubclassOf(Number::class) || this == String::class || this == Char::class || this == Array::class || isSubclassOf(Collection::class)
+internal fun KClass<*>.isPrimitive(): Boolean =
+    isSubclassOf(Number::class) ||
+        this == String::class ||
+        this == Char::class ||
+        this == Array::class ||
+        isSubclassOf(Collection::class)

core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/TypeUtils.kt

@@ -12,7 +12,9 @@ import kotlin.reflect.KType
 import kotlin.reflect.KTypeParameter
 import kotlin.reflect.KTypeProjection
 import kotlin.reflect.KVariance
-import kotlin.reflect.KVariance.*
+import kotlin.reflect.KVariance.IN
+import kotlin.reflect.KVariance.INVARIANT
+import kotlin.reflect.KVariance.OUT
 import kotlin.reflect.KVisibility
 import kotlin.reflect.full.allSuperclasses
 import kotlin.reflect.full.createType
@@ -463,6 +465,9 @@ internal fun guessValueType(values: Sequence<Any?>, upperBound: KType? = null, l
     }
 }
 
+internal val KType.isNothing: Boolean
+    get() = classifier == Nothing::class
+
 internal fun nothingType(nullable: Boolean): KType =
     if (nullable) {
         typeOf<List<Nothing?>>()

core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/aggregation/aggregators/Aggregators.kt

@@ -16,6 +16,9 @@ internal object Aggregators {
     private fun <C : Any, R> mergedValues(aggregate: Iterable<C?>.(KType) -> R?) =
         MergedValuesAggregator.Factory(aggregate, true)
 
+    private fun <C : Any, R> mergedValuesChangingTypes(aggregate: Iterable<C?>.(KType) -> R?) =
+        MergedValuesAggregator.Factory(aggregate, false)
+
     private fun <C, R> changesType(aggregate1: Iterable<C>.(KType) -> R, aggregate2: Iterable<R>.(KType) -> R) =
         TwoStepAggregator.Factory(aggregate1, aggregate2, false)
 
@@ -33,7 +36,7 @@ internal object Aggregators {
     val max by preservesType<Comparable<Any?>> { maxOrNull() }
 
     val std by withOption2<Boolean, Int, Number, Double> { skipNA, ddof ->
-        mergedValues { std(it, skipNA, ddof) }
+        mergedValuesChangingTypes { std(it, skipNA, ddof) }
     }
 
     val mean by withOption<Boolean, Number, Double> { skipNA ->

core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/guess.kt

@@ -35,10 +35,10 @@ public sealed interface SupportedFormat {
 public sealed interface SupportedFormatSample {
 
     @JvmInline
-    public value class File(public val sampleFile: java.io.File) : SupportedFormatSample
+    public value class DataFile(public val sampleFile: File) : SupportedFormatSample
 
     @JvmInline
-    public value class URL(public val sampleUrl: java.net.URL) : SupportedFormatSample
+    public value class DataUrl(public val sampleUrl: URL) : SupportedFormatSample
 
     @JvmInline
     public value class PathString(public val samplePath: String) : SupportedFormatSample
@@ -138,13 +138,13 @@ internal fun guessFormatForExtension(
 internal fun guessFormat(
     file: File,
     formats: List<SupportedFormat> = supportedFormats,
-    sample: SupportedFormatSample.File? = SupportedFormatSample.File(file),
+    sample: SupportedFormatSample.DataFile? = SupportedFormatSample.DataFile(file),
 ): SupportedFormat? = guessFormatForExtension(file.extension.lowercase(), formats, sample = sample)
 
 internal fun guessFormat(
     url: URL,
     formats: List<SupportedFormat> = supportedFormats,
-    sample: SupportedFormatSample.URL? = SupportedFormatSample.URL(url),
+    sample: SupportedFormatSample.DataUrl? = SupportedFormatSample.DataUrl(url),
 ): SupportedFormat? = guessFormatForExtension(url.path.substringAfterLast("."), formats, sample = sample)
 
 internal fun guessFormat(

core/generated-sources/src/test/kotlin/org/jetbrains/kotlinx/dataframe/statistics/std.kt

@@ -2,6 +2,7 @@ package org.jetbrains.kotlinx.dataframe.statistics
 
 import io.kotest.matchers.shouldBe
 import org.jetbrains.kotlinx.dataframe.api.columnOf
+import org.jetbrains.kotlinx.dataframe.api.columnTypes
 import org.jetbrains.kotlinx.dataframe.api.dataFrameOf
 import org.jetbrains.kotlinx.dataframe.api.std
 import org.jetbrains.kotlinx.dataframe.math.std
@@ -21,6 +22,7 @@ class StdTests {
         value.std() shouldBe expected
         df[value].std() shouldBe expected
         df.std { value } shouldBe expected
+        df.std().columnTypes().single() shouldBe typeOf<Double>()
     }
 
     @Test

core/generated-sources/src/test/kotlin/org/jetbrains/kotlinx/dataframe/testSets/person/DataFrameTests.kt

@@ -7,6 +7,7 @@ import io.kotest.matchers.shouldBe
 import io.kotest.matchers.shouldNotBe
 import org.jetbrains.kotlinx.dataframe.AnyFrame
 import org.jetbrains.kotlinx.dataframe.AnyRow
+import org.jetbrains.kotlinx.dataframe.DataColumn
 import org.jetbrains.kotlinx.dataframe.DataFrame
 import org.jetbrains.kotlinx.dataframe.DataRow
 import org.jetbrains.kotlinx.dataframe.RowExpression
@@ -79,6 +80,7 @@ import org.jetbrains.kotlinx.dataframe.api.intoColumns
 import org.jetbrains.kotlinx.dataframe.api.intoList
 import org.jetbrains.kotlinx.dataframe.api.intoRows
 import org.jetbrains.kotlinx.dataframe.api.isColumnGroup
+import org.jetbrains.kotlinx.dataframe.api.isComparable
 import org.jetbrains.kotlinx.dataframe.api.isEmpty
 import org.jetbrains.kotlinx.dataframe.api.isFrameColumn
 import org.jetbrains.kotlinx.dataframe.api.isNA
@@ -119,6 +121,7 @@ import org.jetbrains.kotlinx.dataframe.api.rename
 import org.jetbrains.kotlinx.dataframe.api.reorderColumnsByName
 import org.jetbrains.kotlinx.dataframe.api.replace
 import org.jetbrains.kotlinx.dataframe.api.rows
+import org.jetbrains.kotlinx.dataframe.api.schema
 import org.jetbrains.kotlinx.dataframe.api.select
 import org.jetbrains.kotlinx.dataframe.api.single
 import org.jetbrains.kotlinx.dataframe.api.sortBy
@@ -165,18 +168,22 @@ import org.jetbrains.kotlinx.dataframe.exceptions.ExcessiveColumnsException
 import org.jetbrains.kotlinx.dataframe.exceptions.TypeConversionException
 import org.jetbrains.kotlinx.dataframe.get
 import org.jetbrains.kotlinx.dataframe.hasNulls
+import org.jetbrains.kotlinx.dataframe.impl.DataFrameImpl
 import org.jetbrains.kotlinx.dataframe.impl.DataFrameSize
 import org.jetbrains.kotlinx.dataframe.impl.api.convertToImpl
 import org.jetbrains.kotlinx.dataframe.impl.between
 import org.jetbrains.kotlinx.dataframe.impl.columns.isMissingColumn
 import org.jetbrains.kotlinx.dataframe.impl.emptyPath
 import org.jetbrains.kotlinx.dataframe.impl.getColumnsImpl
+import org.jetbrains.kotlinx.dataframe.impl.isNothing
 import org.jetbrains.kotlinx.dataframe.impl.nothingType
+import org.jetbrains.kotlinx.dataframe.impl.projectTo
 import org.jetbrains.kotlinx.dataframe.impl.trackColumnAccess
 import org.jetbrains.kotlinx.dataframe.index
 import org.jetbrains.kotlinx.dataframe.io.renderValueForStdout
 import org.jetbrains.kotlinx.dataframe.kind
 import org.jetbrains.kotlinx.dataframe.math.mean
+import org.jetbrains.kotlinx.dataframe.name
 import org.jetbrains.kotlinx.dataframe.ncol
 import org.jetbrains.kotlinx.dataframe.nrow
 import org.jetbrains.kotlinx.dataframe.size
@@ -2358,6 +2365,47 @@ class DataFrameTests : BaseTest() {
         desc.print()
     }
 
+    @DataSchema
+    data class ComparableTest(
+        val int: Int,
+        val comparableInt: Comparable<Int>,
+        val string: String,
+        val comparableString: Comparable<String>,
+        val comparableStar: Comparable<*>,
+        val comparableNothing: Comparable<Nothing>,
+    )
+
+    @Test
+    fun `is comparable`() {
+        val df = listOf(
+            ComparableTest(1, 1, "a", "a", 1, 1),
+            ComparableTest(2, 2, "b", "b", "2", "2"),
+        ).toDataFrame()
+
+        df.int.isComparable() shouldBe true
+        df.comparableInt.isComparable() shouldBe true
+        df.string.isComparable() shouldBe true
+        df.comparableString.isComparable() shouldBe true
+        df.comparableStar.isComparable() shouldBe false
+        df.comparableNothing.isComparable() shouldBe false
+    }
+
+    @Test
+    fun `describe twice minimal`() {
+        val df = dataFrameOf("a", "b")(1, "foo", 3, "bar")
+        val desc1 = df.describe()
+        val desc2 = desc1.describe()
+        desc2::class shouldBe DataFrameImpl::class
+    }
+
+    @Test
+    fun `describe twice`() {
+        val df = typed.group { age and weight }.into("info").groupBy { city }.toDataFrame()
+        val desc1 = df.describe()
+        val desc2 = desc1.describe()
+        desc2::class shouldBe DataFrameImpl::class
+    }
+
     @Test
     fun `index by column accessor`() {
         val col = listOf(1, 2, 3, 4, 5).toColumn("name")

core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/DataColumnType.kt

@@ -2,10 +2,13 @@ package org.jetbrains.kotlinx.dataframe.api
 
 import org.jetbrains.kotlinx.dataframe.AnyCol
 import org.jetbrains.kotlinx.dataframe.columns.ColumnKind
+import org.jetbrains.kotlinx.dataframe.impl.isNothing
+import org.jetbrains.kotlinx.dataframe.impl.projectTo
 import org.jetbrains.kotlinx.dataframe.type
 import org.jetbrains.kotlinx.dataframe.typeClass
 import kotlin.reflect.KClass
 import kotlin.reflect.KType
+import kotlin.reflect.KTypeProjection
 import kotlin.reflect.full.isSubclassOf
 import kotlin.reflect.full.isSubtypeOf
 import kotlin.reflect.typeOf
@@ -16,7 +19,8 @@ public fun AnyCol.isFrameColumn(): Boolean = kind() == ColumnKind.Frame
 
 public fun AnyCol.isValueColumn(): Boolean = kind() == ColumnKind.Value
 
-public fun AnyCol.isSubtypeOf(type: KType): Boolean = this.type.isSubtypeOf(type) && (!this.type.isMarkedNullable || type.isMarkedNullable)
+public fun AnyCol.isSubtypeOf(type: KType): Boolean =
+    this.type.isSubtypeOf(type) && (!this.type.isMarkedNullable || type.isMarkedNullable)
 
 public inline fun <reified T> AnyCol.isSubtypeOf(): Boolean = isSubtypeOf(typeOf<T>())
 
@@ -26,9 +30,23 @@ public fun AnyCol.isNumber(): Boolean = isSubtypeOf<Number?>()
 
 public fun AnyCol.isList(): Boolean = typeClass == List::class
 
-public fun AnyCol.isComparable(): Boolean = isSubtypeOf<Comparable<*>?>()
+/**
+ * Returns `true` if [this] column is comparable, i.e. its type is a subtype of [Comparable] and its
+ * type argument is not [Nothing].
+ */
+public fun AnyCol.isComparable(): Boolean =
+    isSubtypeOf<Comparable<*>?>() &&
+        type().projectTo(Comparable::class).arguments[0].let {
+            it != KTypeProjection.STAR &&
+                it.type?.isNothing != true
+        }
 
 @PublishedApi
 internal fun AnyCol.isPrimitive(): Boolean = typeClass.isPrimitive()
 
-internal fun KClass<*>.isPrimitive(): Boolean = isSubclassOf(Number::class) || this == String::class || this == Char::class || this == Array::class || isSubclassOf(Collection::class)
+internal fun KClass<*>.isPrimitive(): Boolean =
+    isSubclassOf(Number::class) ||
+        this == String::class ||
+        this == Char::class ||
+        this == Array::class ||
+        isSubclassOf(Collection::class)

core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/TypeUtils.kt

@@ -12,7 +12,9 @@ import kotlin.reflect.KType
 import kotlin.reflect.KTypeParameter
 import kotlin.reflect.KTypeProjection
 import kotlin.reflect.KVariance
-import kotlin.reflect.KVariance.*
+import kotlin.reflect.KVariance.IN
+import kotlin.reflect.KVariance.INVARIANT
+import kotlin.reflect.KVariance.OUT
 import kotlin.reflect.KVisibility
 import kotlin.reflect.full.allSuperclasses
 import kotlin.reflect.full.createType
@@ -463,6 +465,9 @@ internal fun guessValueType(values: Sequence<Any?>, upperBound: KType? = null, l
     }
 }
 
+internal val KType.isNothing: Boolean
+    get() = classifier == Nothing::class
+
 internal fun nothingType(nullable: Boolean): KType =
     if (nullable) {
         typeOf<List<Nothing?>>()

core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/aggregation/aggregators/Aggregators.kt

@@ -16,6 +16,9 @@ internal object Aggregators {
     private fun <C : Any, R> mergedValues(aggregate: Iterable<C?>.(KType) -> R?) =
         MergedValuesAggregator.Factory(aggregate, true)
 
+    private fun <C : Any, R> mergedValuesChangingTypes(aggregate: Iterable<C?>.(KType) -> R?) =
+        MergedValuesAggregator.Factory(aggregate, false)
+
     private fun <C, R> changesType(aggregate1: Iterable<C>.(KType) -> R, aggregate2: Iterable<R>.(KType) -> R) =
         TwoStepAggregator.Factory(aggregate1, aggregate2, false)
 
@@ -33,7 +36,7 @@ internal object Aggregators {
     val max by preservesType<Comparable<Any?>> { maxOrNull() }
 
     val std by withOption2<Boolean, Int, Number, Double> { skipNA, ddof ->
-        mergedValues { std(it, skipNA, ddof) }
+        mergedValuesChangingTypes { std(it, skipNA, ddof) }
     }
 
     val mean by withOption<Boolean, Number, Double> { skipNA ->