Merge branch 'master' into kproperties-columnsSelectionDsl

# Conflicts:
#	core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/ColumnsSelectionDsl.kt
#	core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/ColumnsSelectionDsl.kt

Author: Jolanrensen

.editorconfig

@@ -11,3 +11,6 @@ indent_size=2
 
 [*.yaml]
 indent_size=2
+
+[*.ipynb]
+insert_final_newline=false

core/build.gradle.kts

@@ -69,10 +69,12 @@ val addGeneratedSourcesToGit by tasks.creating(GitTask::class) {
 
 // Backup the kotlin source files location
 val kotlinMainSources = kotlin.sourceSets.main.get().kotlin.sourceDirectories
+val kotlinTestSources = kotlin.sourceSets.test.get().kotlin.sourceDirectories
 
 // Task to generate the processed documentation
 val processKDocsMain by creatingProcessDocTask(
-    sources = kotlinMainSources.filterNot { "build/generated" in it.path }, // Exclude generated sources
+    sources = (kotlinMainSources + kotlinTestSources) // Include both test and main sources for cross-referencing
+        .filterNot { "build/generated" in it.path }, // Exclude generated sources
 ) {
     target = file(generatedSourcesFolderName)
     processors = listOf(
@@ -107,8 +109,9 @@ tasks.withType<Jar> {
     doFirst {
         kotlin.sourceSets.main {
             kotlin.setSrcDirs(
-                processKDocsMain.targets +
-                    kotlinMainSources.filter { "build/generated" in it.path } // Include generated sources (which were excluded above)
+                processKDocsMain.targets
+                    .filterNot { "src/test/kotlin" in it.path || "src/test/java" in it.path } // filter out test sources again
+                    .plus(kotlinMainSources.filter { "build/generated" in it.path }) // Include generated sources (which were excluded above)
             )
         }
     }

core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/aliases.kt

@@ -74,6 +74,17 @@ public typealias RowValueExpression<T, C, R> = DataRow<T>.(it: C) -> R
  */
 public typealias RowColumnExpression<T, C, R> = (row: DataRow<T>, col: DataColumn<C>) -> R
 
+/**
+ * [ColumnExpression] is used to express or select any instance of `R` using the given instance of [DataColumn]`<C>` as
+ * `this` and `it`.
+ *
+ * Shorthand for:
+ * ```kotlin
+ * DataColumn<C>.(it: DataColumn<C>) -> R
+ * ```
+ */
+public typealias ColumnExpression<C, R> = Selector<DataColumn<C>, R>
+
 /**
  * [ColumnSelector] is used to express or select a single column, represented by [SingleColumn]`<C>`, using the
  * context of [ColumnsSelectionDsl]`<T>` as `this` and `it`.

core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/ColumnsSelectionDsl.kt

@@ -50,7 +50,8 @@ public fun <T> DataFrame<T>.add(vararg columns: AnyBaseCol): DataFrame<T> = addA
  * @throws [UnequalColumnSizesException] if columns in expected result have different sizes
  * @return new [DataFrame] with added columns
  */
-public fun <T> DataFrame<T>.addAll(columns: Iterable<AnyBaseCol>): DataFrame<T> = dataFrameOf(columns() + columns).cast()
+public fun <T> DataFrame<T>.addAll(columns: Iterable<AnyBaseCol>): DataFrame<T> =
+    dataFrameOf(columns() + columns).cast()
 
 /**
  * Creates new [DataFrame] with all columns from given [dataFrames] added to the end of original [DataFrame.columns] list.
@@ -75,7 +76,8 @@ public fun <T> DataFrame<T>.add(vararg dataFrames: AnyFrame): DataFrame<T> = add
  * @return new [DataFrame] with added columns
  */
 @JvmName("addAllFrames")
-public fun <T> DataFrame<T>.addAll(dataFrames: Iterable<AnyFrame>): DataFrame<T> = addAll(dataFrames.flatMap { it.columns() })
+public fun <T> DataFrame<T>.addAll(dataFrames: Iterable<AnyFrame>): DataFrame<T> =
+    addAll(dataFrames.flatMap { it.columns() })
 
 // endregion
 
@@ -96,7 +98,16 @@ public interface AddDataRow<out T> : DataRow<T> {
     public fun <C> AnyRow.newValue(): C
 }
 
-public typealias AddExpression<T, C> = Selector<AddDataRow<T>, C>
+/**
+ * [AddExpression] is used to express or select any instance of `R` using the given instance of [AddDataRow]`<T>` as
+ * `this` and `it`.
+ *
+ * Shorthand for:
+ * ```kotlin
+ * AddDataRow<T>.(it: AddDataRow<T>) -> R
+ * ```
+ */
+public typealias AddExpression<T, R> = Selector<AddDataRow<T>, R>
 
 /**
  * Creates new column using row [expression] and adds it to the end of [DataFrame]
@@ -166,11 +177,15 @@ public class AddDsl<T>(@PublishedApi internal val df: DataFrame<T>) : ColumnsCon
         return df.mapToColumn("", Infer.Nulls, expression)
     }
 
-    public inline infix fun <reified R> String.from(noinline expression: RowExpression<T, R>): Boolean = add(this, Infer.Nulls, expression)
+    public inline infix fun <reified R> String.from(noinline expression: RowExpression<T, R>): Boolean =
+        add(this, Infer.Nulls, expression)
 
     // TODO: use path instead of name
-    public inline infix fun <reified R> ColumnAccessor<R>.from(noinline expression: RowExpression<T, R>): Boolean = name().from(expression)
-    public inline infix fun <reified R> KProperty<R>.from(noinline expression: RowExpression<T, R>): Boolean = add(name, Infer.Nulls, expression)
+    public inline infix fun <reified R> ColumnAccessor<R>.from(noinline expression: RowExpression<T, R>): Boolean =
+        name().from(expression)
+
+    public inline infix fun <reified R> KProperty<R>.from(noinline expression: RowExpression<T, R>): Boolean =
+        add(name, Infer.Nulls, expression)
 
     public infix fun String.from(column: AnyColumnReference): Boolean = add(column.rename(this))
     public inline infix fun <reified R> ColumnAccessor<R>.from(column: ColumnReference<R>): Boolean = name() from column

core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/Nulls.kt

@@ -2,29 +2,43 @@ package org.jetbrains.kotlinx.dataframe.api
 
 import org.jetbrains.kotlinx.dataframe.ColumnsSelector
 import org.jetbrains.kotlinx.dataframe.DataFrame
-import org.jetbrains.kotlinx.dataframe.Predicate
+import org.jetbrains.kotlinx.dataframe.RowValueFilter
 import org.jetbrains.kotlinx.dataframe.columns.ColumnAccessor
 import org.jetbrains.kotlinx.dataframe.columns.ColumnReference
-import org.jetbrains.kotlinx.dataframe.impl.and
 import org.jetbrains.kotlinx.dataframe.impl.api.gatherImpl
 import org.jetbrains.kotlinx.dataframe.impl.columnName
 import org.jetbrains.kotlinx.dataframe.impl.columns.toColumns
 import kotlin.reflect.KProperty
 import kotlin.reflect.KType
 import kotlin.reflect.typeOf
 
-public fun <T, C> DataFrame<T>.gather(selector: ColumnsSelector<T, C>): Gather<T, C, String, C> = Gather(
-    this, selector, null, typeOf<String>(),
-    { it }, null
-)
-public fun <T> DataFrame<T>.gather(vararg columns: String): Gather<T, Any?, String, Any?> = gather { columns.toColumns() }
-public fun <T, C> DataFrame<T>.gather(vararg columns: ColumnReference<C>): Gather<T, C, String, C> = gather { columns.toColumns() }
-public fun <T, C> DataFrame<T>.gather(vararg columns: KProperty<C>): Gather<T, C, String, C> = gather { columns.toColumns() }
+public fun <T, C> DataFrame<T>.gather(selector: ColumnsSelector<T, C>): Gather<T, C, String, C> =
+    Gather(
+        df = this,
+        columns = selector,
+        filter = null,
+        keyType = typeOf<String>(),
+        keyTransform = { it },
+        valueTransform = null,
+    )
 
-public fun <T, C, K, R> Gather<T, C, K, R>.where(filter: Predicate<C>): Gather<T, C, K, R> = copy(filter = this.filter and filter)
-public fun <T, C, K, R> Gather<T, C?, K, R>.notNull(): Gather<T, C, K, R> = where { it != null } as Gather<T, C, K, R>
+public fun <T> DataFrame<T>.gather(vararg columns: String): Gather<T, Any?, String, Any?> =
+    gather { columns.toColumns() }
 
-public fun <T, C, K, R> Gather<T, C, K, R>.explodeLists(): Gather<T, C, K, R> = copy(explode = true)
+public fun <T, C> DataFrame<T>.gather(vararg columns: ColumnReference<C>): Gather<T, C, String, C> =
+    gather { columns.toColumns() }
+
+public fun <T, C> DataFrame<T>.gather(vararg columns: KProperty<C>): Gather<T, C, String, C> =
+    gather { columns.toColumns() }
+
+public fun <T, C, K, R> Gather<T, C, K, R>.where(filter: RowValueFilter<T, C>): Gather<T, C, K, R> =
+    copy(filter = this.filter and filter)
+
+public fun <T, C, K, R> Gather<T, C?, K, R>.notNull(): Gather<T, C, K, R> =
+    where { it != null } as Gather<T, C, K, R>
+
+public fun <T, C, K, R> Gather<T, C, K, R>.explodeLists(): Gather<T, C, K, R> =
+    copy(explode = true)
 
 public inline fun <T, C, reified K, R> Gather<T, C, *, R>.mapKeys(noinline transform: (String) -> K): Gather<T, C, K, R> =
     copy(keyTransform = transform as ((String) -> Nothing), keyType = typeOf<K>()) as Gather<T, C, K, R>
@@ -35,7 +49,7 @@ public fun <T, C, K, R> Gather<T, C, K, *>.mapValues(transform: (C) -> R): Gathe
 public data class Gather<T, C, K, R>(
     internal val df: DataFrame<T>,
     internal val columns: ColumnsSelector<T, C>,
-    internal val filter: ((C) -> Boolean)? = null,
+    internal val filter: RowValueFilter<T, C>? = null,
     internal val keyType: KType? = null,
     internal val keyTransform: ((String) -> K),
     internal val valueTransform: ((C) -> R)? = null,

core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/add.kt

@@ -1,8 +1,8 @@
 package org.jetbrains.kotlinx.dataframe.api
 
 import org.jetbrains.kotlinx.dataframe.AnyCol
+import org.jetbrains.kotlinx.dataframe.ColumnExpression
 import org.jetbrains.kotlinx.dataframe.ColumnsSelector
-import org.jetbrains.kotlinx.dataframe.DataColumn
 import org.jetbrains.kotlinx.dataframe.DataFrame
 import org.jetbrains.kotlinx.dataframe.Selector
 import org.jetbrains.kotlinx.dataframe.columns.ColumnReference
@@ -21,22 +21,28 @@ public data class Reorder<T, C>(
 }
 
 public fun <T, C> DataFrame<T>.reorder(selector: ColumnsSelector<T, C>): Reorder<T, C> = Reorder(this, selector, false)
-public fun <T, C> DataFrame<T>.reorder(vararg columns: ColumnReference<C>): Reorder<T, C> = reorder { columns.toColumns() }
+public fun <T, C> DataFrame<T>.reorder(vararg columns: ColumnReference<C>): Reorder<T, C> =
+    reorder { columns.toColumns() }
+
 public fun <T, C> DataFrame<T>.reorder(vararg columns: KProperty<C>): Reorder<T, C> = reorder { columns.toColumns() }
 public fun <T> DataFrame<T>.reorder(vararg columns: String): Reorder<T, *> = reorder { columns.toColumns() }
 
-public fun <T, C, V : Comparable<V>> Reorder<T, C>.by(expression: Selector<DataColumn<C>, V>): DataFrame<T> = reorderImpl(false, expression)
+public fun <T, C, V : Comparable<V>> Reorder<T, C>.by(expression: ColumnExpression<C, V>): DataFrame<T> =
+    reorderImpl(false, expression)
 
-public fun <T, C> Reorder<T, C>.byName(desc: Boolean = false): DataFrame<T> = if (desc) byDesc { it.name } else by { it.name }
+public fun <T, C> Reorder<T, C>.byName(desc: Boolean = false): DataFrame<T> =
+    if (desc) byDesc { it.name } else by { it.name }
 
-public fun <T, C, V : Comparable<V>> Reorder<T, C>.byDesc(expression: Selector<DataColumn<C>, V>): DataFrame<T> = reorderImpl(true, expression)
+public fun <T, C, V : Comparable<V>> Reorder<T, C>.byDesc(expression: ColumnExpression<C, V>): DataFrame<T> =
+    reorderImpl(true, expression)
 
 public fun <T, V : Comparable<V>> DataFrame<T>.reorderColumnsBy(
     dfs: Boolean = true,
     desc: Boolean = false,
     expression: Selector<AnyCol, V>
 ): DataFrame<T> = Reorder(this, { if (dfs) allDfs(true) else all() }, dfs).reorderImpl(desc, expression)
 
-public fun <T> DataFrame<T>.reorderColumnsByName(dfs: Boolean = true, desc: Boolean = false): DataFrame<T> = reorderColumnsBy(dfs, desc) { name() }
+public fun <T> DataFrame<T>.reorderColumnsByName(dfs: Boolean = true, desc: Boolean = false): DataFrame<T> =
+    reorderColumnsBy(dfs, desc) { name() }
 
 // endregion

core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/gather.kt

@@ -10,12 +10,28 @@ import kotlin.reflect.KProperty
 
 // region DataFrame
 
-public fun <T> DataFrame<T>.select(columns: ColumnsSelector<T, *>): DataFrame<T> = get(columns).toDataFrame().cast()
-public fun <T> DataFrame<T>.select(vararg columns: KProperty<*>): DataFrame<T> = select(columns.map { it.columnName })
-public fun <T> DataFrame<T>.select(vararg columns: String): DataFrame<T> = select(columns.asIterable())
-public fun <T> DataFrame<T>.select(vararg columns: AnyColumnReference): DataFrame<T> = select { columns.toColumns() }
-@JvmName("selectT")
-public fun <T> DataFrame<T>.select(columns: Iterable<String>): DataFrame<T> = columns.map { get(it) }.toDataFrame().cast()
-public fun <T> DataFrame<T>.select(columns: Iterable<AnyColumnReference>): DataFrame<T> = select { columns.toColumnSet() }
+public fun <T> DataFrame<T>.select(columns: ColumnsSelector<T, *>): DataFrame<T> =
+    get(columns).toDataFrame().cast()
+
+public fun <T> DataFrame<T>.select(vararg columns: KProperty<*>): DataFrame<T> =
+    select(columns.asIterable())
+
+@JvmName("selectKPropertyIterable")
+public fun <T> DataFrame<T>.select(columns: Iterable<KProperty<*>>): DataFrame<T> =
+    select(columns.map { it.columnName })
+
+public fun <T> DataFrame<T>.select(vararg columns: String): DataFrame<T> =
+    select(columns.asIterable())
+
+@JvmName("selectStringIterable")
+public fun <T> DataFrame<T>.select(columns: Iterable<String>): DataFrame<T> =
+    columns.map { get(it) }.toDataFrame().cast()
+
+public fun <T> DataFrame<T>.select(vararg columns: AnyColumnReference): DataFrame<T> =
+    select { columns.toColumns() }
+
+@JvmName("selectAnyColumnReferenceIterable")
+public fun <T> DataFrame<T>.select(columns: Iterable<AnyColumnReference>): DataFrame<T> =
+    select { columns.toColumnSet() }
 
 // endregion

core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/reorder.kt

@@ -7,6 +7,7 @@ import org.jetbrains.kotlinx.dataframe.ColumnsSelector
 import org.jetbrains.kotlinx.dataframe.DataColumn
 import org.jetbrains.kotlinx.dataframe.DataFrame
 import org.jetbrains.kotlinx.dataframe.DataRow
+import org.jetbrains.kotlinx.dataframe.annotations.DataSchema
 import org.jetbrains.kotlinx.dataframe.columns.ColumnPath
 import org.jetbrains.kotlinx.dataframe.impl.ColumnNameGenerator
 import org.jetbrains.kotlinx.dataframe.impl.api.createDataFrameImpl
@@ -23,7 +24,8 @@ public inline fun <reified T> Iterable<T>.toDataFrame(): DataFrame<T> = toDataFr
     properties()
 }
 
-public inline fun <reified T> Iterable<T>.toDataFrame(noinline body: CreateDataFrameDsl<T>.() -> Unit): DataFrame<T> = createDataFrameImpl(T::class, body)
+public inline fun <reified T> Iterable<T>.toDataFrame(noinline body: CreateDataFrameDsl<T>.() -> Unit): DataFrame<T> =
+    createDataFrameImpl(T::class, body)
 
 public inline fun <reified T> Iterable<T>.toDataFrame(vararg props: KProperty<*>, maxDepth: Int = 0): DataFrame<T> =
     toDataFrame {
@@ -76,6 +78,7 @@ public fun <T> Iterable<Pair<ColumnPath, AnyBaseCol>>.toDataFrameFromPairs(): Da
         when (path.size) {
             0 -> {
             }
+
             1 -> {
                 val name = path[0]
                 val uniqueName = nameGenerator.addUnique(name)
@@ -85,6 +88,7 @@ public fun <T> Iterable<Pair<ColumnPath, AnyBaseCol>>.toDataFrameFromPairs(): Da
                 columns.add(col.rename(uniqueName))
                 columnIndices[uniqueName] = index
             }
+
             else -> {
                 val name = path[0]
                 val uniqueName = columnGroupName.getOrPut(name) {
@@ -187,6 +191,83 @@ public abstract class CreateDataFrameDsl<T> : TraversePropertiesDsl {
 
 // endregion
 
+// region toDataFrame overloads for built-in types
+
+/*
+Without overloads Iterable<String>.toDataFrame produces unexpected result
+
+
+```
+val string = listOf("aaa", "aa", null)
+string.toDataFrame()
+```
+=>
+  length
+0    3
+1    2
+2 null
+ */
+
+@JvmName("toDataFrameByte")
+public inline fun <reified B : Byte?> Iterable<B>.toDataFrame(): DataFrame<ValueProperty<B>> = toDataFrame {
+    ValueProperty<B>::value from { it }
+}.cast()
+
+@JvmName("toDataFrameShort")
+public inline fun <reified S : Short?> Iterable<S>.toDataFrame(): DataFrame<ValueProperty<S>> = toDataFrame {
+    ValueProperty<S>::value from { it }
+}.cast()
+
+@JvmName("toDataFrameInt")
+public inline fun <reified I : Int?> Iterable<I>.toDataFrame(): DataFrame<ValueProperty<I>> = toDataFrame {
+    ValueProperty<I>::value from { it }
+}.cast()
+
+@JvmName("toDataFrameLong")
+public inline fun <reified L : Long?> Iterable<L>.toDataFrame(): DataFrame<ValueProperty<L>> = toDataFrame {
+    ValueProperty<L>::value from { it }
+}.cast()
+
+@JvmName("toDataFrameString")
+public inline fun <reified S : String?> Iterable<S>.toDataFrame(): DataFrame<ValueProperty<S>> = toDataFrame {
+    ValueProperty<S>::value from { it }
+}.cast()
+
+@JvmName("toDataFrameChar")
+public inline fun <reified C : Char?> Iterable<C>.toDataFrame(): DataFrame<ValueProperty<C>> = toDataFrame {
+    ValueProperty<C>::value from { it }
+}.cast()
+
+@JvmName("toDataFrameBoolean")
+public inline fun <reified B : Boolean?> Iterable<B>.toDataFrame(): DataFrame<ValueProperty<B>> = toDataFrame {
+    ValueProperty<B>::value from { it }
+}.cast()
+
+@JvmName("toDataFrameUByte")
+public inline fun <reified U : UByte?> Iterable<U>.toDataFrame(): DataFrame<ValueProperty<U>> = toDataFrame {
+    ValueProperty<U>::value from { it }
+}.cast()
+
+@JvmName("toDataFrameUShort")
+public inline fun <reified U : UShort?> Iterable<U>.toDataFrame(): DataFrame<ValueProperty<U>> = toDataFrame {
+    ValueProperty<U>::value from { it }
+}.cast()
+
+@JvmName("toDataFrameUInt")
+public inline fun <reified U : UInt?> Iterable<U>.toDataFrame(): DataFrame<ValueProperty<U>> = toDataFrame {
+    ValueProperty<U>::value from { it }
+}.cast()
+
+@JvmName("toDataFrameULong")
+public inline fun <reified U : ULong?> Iterable<U>.toDataFrame(): DataFrame<ValueProperty<U>> = toDataFrame {
+    ValueProperty<U>::value from { it }
+}.cast()
+
+@DataSchema
+public interface ValueProperty<T> {
+    public val value: T
+}
+
 // region Create DataFrame from Map
 
 public fun Map<String, Iterable<Any?>>.toDataFrame(): AnyFrame {
@@ -195,7 +276,12 @@ public fun Map<String, Iterable<Any?>>.toDataFrame(): AnyFrame {
 
 @JvmName("toDataFrameColumnPathAnyNullable")
 public fun Map<ColumnPath, Iterable<Any?>>.toDataFrame(): AnyFrame {
-    return map { it.key to DataColumn.createWithTypeInference(it.key.last(), it.value.asList()) }.toDataFrameFromPairs<Unit>()
+    return map {
+        it.key to DataColumn.createWithTypeInference(
+            it.key.last(),
+            it.value.asList()
+        )
+    }.toDataFrameFromPairs<Unit>()
 }
 
 // endregion