Merge branch 'master' into convert_kdocs

# Conflicts:
#	core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/convert.kt

Author: AndreiKingsley

core/api/core.api

@@ -64,6 +64,7 @@ dependencies {
     kotlinCompilerPluginClasspathSamples(projects.plugins.expressionsConverter)
 
     api(libs.commonsCsv)
+
     implementation(libs.commonsIo)
     implementation(libs.serialization.core)
     implementation(libs.serialization.json)
@@ -82,6 +83,9 @@ dependencies {
     testImplementation(libs.jsoup)
     testImplementation(libs.sl4jsimple)
 
+    // for checking results
+    testImplementation(libs.commonsStatisticsDescriptive)
+
     // for samples.api
     testImplementation(projects.dataframeCsv)
 }

core/build.gradle.kts

@@ -16,3 +16,9 @@ internal val skipNaNDefault: Boolean = false
  */
 @PublishedApi
 internal val ddofDefault: Int = 1
+
+/**
+ * whether to skip nulls and NaNs in the cumSum operation.
+ */
+@PublishedApi
+internal val defaultCumSumSkipNA: Boolean = true

core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/Defaults.kt

@@ -13,6 +13,7 @@ import org.jetbrains.kotlinx.dataframe.annotations.AccessApiOverload
 import org.jetbrains.kotlinx.dataframe.annotations.Interpretable
 import org.jetbrains.kotlinx.dataframe.annotations.Refine
 import org.jetbrains.kotlinx.dataframe.columns.ColumnAccessor
+import org.jetbrains.kotlinx.dataframe.columns.ColumnGroup
 import org.jetbrains.kotlinx.dataframe.columns.ColumnPath
 import org.jetbrains.kotlinx.dataframe.columns.ColumnReference
 import org.jetbrains.kotlinx.dataframe.columns.FrameColumn
@@ -269,6 +270,15 @@ public inline fun <reified T> column(values: Iterable<T>): DataColumn<T> =
         allColsMakesColGroup = true,
     ).forceResolve()
 
+@Refine
+@Interpretable("ColumnOfPairs")
+public fun columnOf(vararg columns: Pair<String, AnyBaseCol>): ColumnGroup<*> =
+    dataFrameOf(
+        columns.map { (name, col) ->
+            col.rename(name)
+        },
+    ).asColumnGroup()
+
 // endregion
 
 // region create DataFrame
@@ -290,6 +300,12 @@ public fun dataFrameOf(columns: Iterable<AnyBaseCol>): DataFrame<*> {
     return DataFrameImpl<Unit>(cols, nrow)
 }
 
+@Refine
+@JvmName("dataFrameOfColumns")
+@Interpretable("DataFrameOfPairs")
+public fun dataFrameOf(vararg columns: Pair<String, AnyBaseCol>): DataFrame<*> =
+    dataFrameOf(columns.map { (name, col) -> col.rename(name) })
+
 public fun dataFrameOf(vararg header: ColumnReference<*>): DataFrameBuilder = DataFrameBuilder(header.map { it.name() })
 
 public fun dataFrameOf(vararg columns: AnyBaseCol): DataFrame<*> = dataFrameOf(columns.asIterable())

core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/constructors.kt

@@ -21,6 +21,7 @@ import org.jetbrains.kotlinx.dataframe.annotations.Converter
 import org.jetbrains.kotlinx.dataframe.annotations.HasSchema
 import org.jetbrains.kotlinx.dataframe.annotations.Interpretable
 import org.jetbrains.kotlinx.dataframe.annotations.Refine
+import org.jetbrains.kotlinx.dataframe.columns.BaseColumn
 import org.jetbrains.kotlinx.dataframe.columns.ColumnGroup
 import org.jetbrains.kotlinx.dataframe.columns.ColumnReference
 import org.jetbrains.kotlinx.dataframe.columns.toColumnSet
@@ -96,7 +97,10 @@ public inline fun <T, C, reified R> Convert<T, C?>.notNull(
     }
 
 @HasSchema(schemaArg = 0)
-public class Convert<T, out C>(internal val df: DataFrame<T>, internal val columns: ColumnsSelector<T, C>) {
+public class Convert<T, out C>(
+    @PublishedApi internal val df: DataFrame<T>,
+    @PublishedApi internal val columns: ColumnsSelector<T, C>,
+) {
     public fun <R> cast(): Convert<T, R> = Convert(df, columns as ColumnsSelector<T, R>)
 
     @Refine
@@ -128,6 +132,21 @@ public fun <T, C, R> Convert<T, DataRow<C>>.asFrame(
     body: ColumnsContainer<T>.(ColumnGroup<C>) -> DataFrame<R>,
 ): DataFrame<T> = to { body(this, it.asColumnGroup()).asColumnGroup(it.name()) }
 
+/**
+ * Compiler plugin-friendly variant of [ReplaceClause.with]
+ * [ReplaceClause.with] allows to change both column types and names.
+ * Tracking of column name changes in arbitrary lambda expression is unreliable and generally impossible
+ * to do statically.
+ * This function ensures that all column names remain as is and only their type changes to [R]
+ * Example:
+ * `df.convert { colsOf<String>() }.asColumn { it.asList().parallelStream().map { heavyIO(it) }.toList().toColumn() }`
+ */
+@Refine
+@Interpretable("ConvertAsColumn")
+public inline fun <T, C, R> Convert<T, C>.asColumn(
+    crossinline columnConverter: DataFrame<T>.(DataColumn<C>) -> BaseColumn<R>,
+): DataFrame<T> = df.replace(columns).with { columnConverter(df, it).rename(it.name()) }
+
 @Refine
 @Interpretable("PerRowCol")
 public inline fun <T, C, reified R> Convert<T, C>.perRowCol(