Merge branch 'master' into devcrocod/serialization-patch0

# Conflicts:
#	core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/json.kt
#	core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/JupyterHtmlRenderer.kt
#	core/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/json.kt
#	core/src/test/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/RenderingTests.kt

Author: devcrocod

.github/workflows/gradle-wrapper-validation.yml

@@ -0,0 +1,10 @@
+name: "Validate Gradle Wrapper"
+on: [push, pull_request]
+
+jobs:
+  validation:
+    name: "Validation"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: gradle/wrapper-validation-action@v2

.github/workflows/main.yml

@@ -17,7 +17,7 @@ env:
   ALGOLIA_INDEX_NAME: prod_DATAFRAME_HELP
   ALGOLIA_KEY: ${{ secrets.ALGOLIA_KEY }}
   CONFIG_JSON_PRODUCT: Dataframe
-  CONFIG_JSON_VERSION: '0.12'
+  CONFIG_JSON_VERSION: '0.13'
 
 jobs:
   build-job:
@@ -33,15 +33,15 @@ jobs:
           apt-get update
           apt-get install -y zip
           mkdir -p artifacts
-      - name: Build docs and include sitemap.xml
+      - name: Build docs # and include sitemap.xml TODO: uncomment when sitemap.xml is available
         run: |
           export DISPLAY=:99
           Xvfb :99 &
           /opt/builder/bin/idea.sh helpbuilderinspect -source-dir . -product $PRODUCT --runner github -output-dir artifacts/ || true
           test -e artifacts/$ARTIFACT && echo $ARTIFACT exists
-          cp docs/StardustDocs/sitemap.xml artifacts/sitemap.xml
-          cd artifacts
-          zip -r $ARTIFACT sitemap.xml
+#          cp docs/StardustDocs/sitemap.xml artifacts/sitemap.xml
+#          cd artifacts
+#          zip -r $ARTIFACT sitemap.xml
         working-directory: ${{ github.workspace }}
       - name: Upload modified documentation artifact
         uses: actions/upload-artifact@v3

.gitignore

@@ -1,5 +1,6 @@
 # Default ignored files
 .idea
 .gradle
+.kotlin
 build
 .ipynb_checkpoints

README.md

@@ -1,12 +1,14 @@
-# Kotlin Dataframe: typesafe in-memory structured data processing for JVM
+# Kotlin DataFrame: typesafe in-memory structured data processing for JVM
 [![JetBrains incubator project](https://jb.gg/badges/incubator.svg)](https://confluence.jetbrains.com/display/ALL/JetBrains+on+GitHub)
 [![Kotlin component alpha stability](https://img.shields.io/badge/project-alpha-kotlin.svg?colorA=555555&colorB=DB3683&label=&logo=kotlin&logoColor=ffffff&logoWidth=10)](https://kotlinlang.org/docs/components-stability.html)
-[![Kotlin](https://img.shields.io/badge/kotlin-1.8.20-blue.svg?logo=kotlin)](http://kotlinlang.org)
-[![Maven Central](https://img.shields.io/maven-central/v/org.jetbrains.kotlinx/dataframe?color=blue&label=Maven%20Central)](https://search.maven.org/artifact/org.jetbrains.kotlinx/dataframe)
+[![Kotlin](https://img.shields.io/badge/kotlin-1.9.22-blue.svg?logo=kotlin)](http://kotlinlang.org)
+[![Dynamic XML Badge](https://img.shields.io/badge/dynamic/xml?url=https%3A%2F%2Frepo1.maven.org%2Fmaven2%2Forg%2Fjetbrains%2Fkotlinx%2Fdataframe%2Fmaven-metadata.xml&query=%2F%2Fversion%5Bnot%28contains%28text%28%29%2C%22dev%22%29%29%5D%5Blast%28%29%5D&label=Release%20version)](https://search.maven.org/artifact/org.jetbrains.kotlinx/dataframe)
+[![Dynamic XML Badge](https://img.shields.io/badge/dynamic/xml?url=https%3A%2F%2Frepo1.maven.org%2Fmaven2%2Forg%2Fjetbrains%2Fkotlinx%2Fdataframe%2Fmaven-metadata.xml&query=%2F%2Fversion%5Bcontains%28text%28%29%2C%22dev%22%29%5D%5Blast%28%29%5D&label=Dev%20version&color=yellow
+)](https://search.maven.org/artifact/org.jetbrains.kotlinx/dataframe)
 [![GitHub License](https://img.shields.io/badge/license-Apache%20License%202.0-blue.svg?style=flat)](http://www.apache.org/licenses/LICENSE-2.0)
 [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/Kotlin/dataframe/HEAD)
 
-Kotlin Dataframe aims to reconcile Kotlin's static typing with the dynamic nature of data by utilizing both the full power of the Kotlin language and the opportunities provided by intermittent code execution in Jupyter notebooks and REPL.   
+Kotlin DataFrame aims to reconcile Kotlin's static typing with the dynamic nature of data by utilizing both the full power of the Kotlin language and the opportunities provided by intermittent code execution in Jupyter notebooks and REPL.   
 
 * **Hierarchical** — represents hierarchical data structures, such as JSON or a tree of JVM objects.
 * **Functional** — data processing pipeline is organized in a chain of `DataFrame` transformation operations. Every operation returns a new instance of `DataFrame` reusing underlying storage wherever it's possible.
@@ -16,7 +18,7 @@ Kotlin Dataframe aims to reconcile Kotlin's static typing with the dynamic natur
 * **Interoperable** — convertable with Kotlin data classes and collections.
 * **Generic** — can store objects of any type, not only numbers or strings.
 * **Typesafe** — on-the-fly generation of extension properties for type safe data access with Kotlin-style care for null safety.
-* **Polymorphic** — type compatibility derives from column schema compatibility. You can define a function that requires a special subset of columns in dataframe but doesn't care about other columns.
+* **Polymorphic** — type compatibility derives from column schema compatibility. You can define a function that requires a special subset of columns in a dataframe but doesn't care about other columns.
 
 Integrates with [Kotlin kernel for Jupyter](https://github.com/Kotlin/kotlin-jupyter). Inspired by [krangl](https://github.com/holgerbrandl/krangl), Kotlin Collections and [pandas](https://pandas.pydata.org/)
 
@@ -31,20 +33,20 @@ You could find the following articles there:
 * [Full list of all supported operations](https://kotlin.github.io/dataframe/operations.html)
     * [Reading from SQL databases](https://kotlin.github.io/dataframe/readsqldatabases.html)
     * [Reading/writing from/to different file formats like JSON, CSV, Apache Arrow](https://kotlin.github.io/dataframe/read.html)
-    * [Joining a few dataframes](https://kotlin.github.io/dataframe/join.html)
+    * [Joining dataframes](https://kotlin.github.io/dataframe/join.html)
     * [GroupBy operation](https://kotlin.github.io/dataframe/groupby.html)
 * [Rendering to HTML](https://kotlin.github.io/dataframe/tohtml.html#jupyter-notebooks)
 
 ## Setup
 
 ```kotlin
-implementation("org.jetbrains.kotlinx:dataframe:0.12.1")
+implementation("org.jetbrains.kotlinx:dataframe:0.13.1")
 ```
 
 Optional Gradle plugin for enhanced type safety and schema generation
 https://kotlin.github.io/dataframe/schemasgradle.html
 ```kotlin
-id("org.jetbrains.kotlinx.dataframe") version "0.12.1"
+id("org.jetbrains.kotlinx.dataframe") version "0.13.1"
 ```
 
 Check out the [custom setup page](https://kotlin.github.io/dataframe/gettingstartedgradleadvanced.html) if you don't need some of the formats as dependencies,
@@ -69,7 +71,7 @@ df.filter { "stargazers_count"<Int>() > 50 }.print()
 
 Requires Gradle plugin to work
 ```kotlin
-id("org.jetbrains.kotlinx.dataframe") version "0.12.1"
+id("org.jetbrains.kotlinx.dataframe") version "0.13.1"
 ```
 
 Plugin generates extension properties API for provided sample of data. Column names and their types become discoverable in completion.
@@ -97,13 +99,13 @@ fun main() {
 
 ## Getting started in Jupyter Notebook / Kotlin Notebook
 
-Install [Kotlin kernel](https://github.com/Kotlin/kotlin-jupyter) for [Jupyter](https://jupyter.org/)
+Install the [Kotlin kernel](https://github.com/Kotlin/kotlin-jupyter) for [Jupyter](https://jupyter.org/)
 
-Import stable `dataframe` version into notebook: 
+Import the stable `dataframe` version into a notebook: 
 ```
 %use dataframe
 ```
-or specific version:
+or a specific version:
 ```
 %use dataframe(<version>)
 ```
@@ -203,7 +205,7 @@ clean
     }
 ```
 
-Check it out on [**Datalore**](https://datalore.jetbrains.com/view/notebook/vq5j45KWkYiSQnACA2Ymij) to get a better visual impression of what happens and what the hierarchical DataFrame structure looks like. 
+Check it out on [**Datalore**](https://datalore.jetbrains.com/view/notebook/vq5j45KWkYiSQnACA2Ymij) to get a better visual impression of what happens and what the hierarchical dataframe structure looks like. 
 
 Explore [**more examples here**](examples).
 
@@ -219,12 +221,12 @@ This table shows the mapping between main library component versions and minimum
 | 0.11.1                   | 8                    | 1.8.20         | 0.11.0-358             | 3.0.0           | 11.0.0               |
 | 0.12.0                   | 8                    | 1.9.0          | 0.11.0-358             | 3.0.0           | 11.0.0               |
 | 0.12.1                   | 8                    | 1.9.0          | 0.11.0-358             | 3.0.0           | 11.0.0               |
-
+| 0.13.1                   | 8                    | 1.9.22         | 0.12.0-139             | 3.0.0           | 15.0.0               |
 
 ## Code of Conduct
 
 This project and the corresponding community are governed by the [JetBrains Open Source and Community Code of Conduct](https://confluence.jetbrains.com/display/ALL/JetBrains+Open+Source+and+Community+Code+of+Conduct). Please make sure you read it.
 
 ## License
 
-Kotlin Dataframe is licensed under the [Apache 2.0 License](LICENSE).
+Kotlin DataFrame is licensed under the [Apache 2.0 License](LICENSE).

RELEASE_CHECK_LIST.md

@@ -2,6 +2,9 @@
 
 1. Run code inspections (fix typos, Kotlin issues, fix code formatting, linter). **RC**
 2. Write missed KDocs for new APIs. **RC**
+3. Update documentation on Kotlin site **RC**
+   - The overview [page](https://kotlinlang.org/docs/data-analysis-overview.html)
+   - The Data Analysis subchapter, for example, the [page](https://kotlinlang.org/docs/data-analysis-work-with-data-sources.html)
 3. Update tutorials according to the latest code changes.
 4. Update README.MD according last code changes:
    - update an artifact version.
@@ -23,16 +26,17 @@
     - Check on Datalore with a test project (TODO: add link).
     - Check for Android with a test project (TODO: add link).
     - Check for ServerSide with a test project (TODO: add link).
-15. Publish Documentation from [GitHub Action](https://github.com/Kotlin/dataframe/actions/workflows/main.yml)
-16. Prepare and publish the Release Notes. **RC**
-17. Create a Release from the release tag on GitHub. **RC**
-18. Update a KDF version in the [Kotlin Jupyter Descriptor](https://github.com/Kotlin/kotlin-jupyter-libraries/blob/master/dataframe.json). Now the Renovate bot does this.
-19. Update the DataFrame version in the `gradle.properties` file for the next release cycle (i.e. 0.10.0 -> 0.11.0)
-20. Update deprecated functions in [deprecationMessages.kt](/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/util/deprecationMessages.kt)
+15. Run `./gradlew korro` to update the code snippets for the docs.
+16. Publish Documentation from [GitHub Action](https://github.com/Kotlin/dataframe/actions/workflows/main.yml)
+17. Prepare and publish the Release Notes. **RC**
+18. Create a Release from the release tag on GitHub. **RC**
+19. Update a KDF version in the [Kotlin Jupyter Descriptor](https://github.com/Kotlin/kotlin-jupyter-libraries/blob/master/dataframe.json). Now the Renovate bot does this.
+20. Update the DataFrame version in the `gradle.properties` file for the next release cycle (i.e. 0.10.0 -> 0.11.0)
+21. Update deprecated functions in [deprecationMessages.kt](/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/util/deprecationMessages.kt)
     such that 
     - `Level.WARNING` messages are changed to `Level.ERROR`
     - `Level.ERROR` messages and their functions are removed.
     - Update regions in the file accordingly.
-21. Update Notebook examples, both in the project and on Datalore.
+22. Update Notebook examples, both in the project and on Datalore.
 
 (Activities that need to be done for **R**elease **C**andidate releases are marked as such)

build.gradle.kts

@@ -233,7 +233,11 @@ kotlinPublications {
             apache2()
         }
         developers {
-            developer("nikitinas", "Anatoly Nikitin", "[email protected]")
+            developer("koperagen", "Nikita Klimenko", "[email protected]")
+            developer("Jolanrensen", "Jolan Rensen", "[email protected]")
+            developer("zaleslaw", "Aleksei Zinovev", "[email protected]")
+            developer("ermolenkodev", "Nikita Ermolenko", "[email protected]")
+            developer("nikitinas", "Anatoly Nikitin", "[email protected]")
         }
     }
 

core/build.gradle.kts

@@ -6,6 +6,7 @@ import nl.jolanrensen.docProcessor.gradle.creatingProcessDocTask
 import org.gradle.jvm.tasks.Jar
 import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
 import org.jmailen.gradle.kotlinter.tasks.LintTask
+import xyz.ronella.gradle.plugin.simple.git.OSType
 import xyz.ronella.gradle.plugin.simple.git.task.GitTask
 
 plugins {
@@ -137,7 +138,8 @@ val clearSamplesOutputs by tasks.creating {
 
     doFirst {
         delete {
-            delete(fileTree(File(projectDir, "../docs/StardustDocs/snippets")))
+            val generatedSnippets = fileTree(file("../docs/StardustDocs/snippets")).exclude("**/manual/**")
+            delete(generatedSnippets)
         }
     }
 }
@@ -175,10 +177,17 @@ val installGitPreCommitHook by tasks.creating(Copy::class) {
         from(File(rootProject.rootDir, "gradle/scripts/pre-commit"))
         into(gitHooksDir)
         fileMode = 755
+
+        // Workaround for https://github.com/Kotlin/dataframe/issues/612
+        if (OSType.identify() in listOf(OSType.Mac, OSType.Linux)) doLast {
+            exec {
+                workingDir(gitHooksDir)
+                commandLine("chmod", "755", "pre-commit")
+            }
+        }
     } else {
         logger.lifecycle("'.git/hooks' directory not found. Skipping installation of pre-commit hook.")
     }
-
 }
 tasks.named("assemble") {
     dependsOn(installGitPreCommitHook)

core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/aggregation/ColumnsForAggregateSelectionDsl.kt

@@ -13,17 +13,19 @@ public interface ColumnsForAggregateSelectionDsl<out T> : ColumnsSelectionDsl<T>
         ConfiguredAggregateColumn.withDefault(this, defaultValue)
 
     public infix fun <C> SingleColumn<C>.default(defaultValue: C): SingleColumn<C> =
-        ConfiguredAggregateColumn.withDefault(this, defaultValue).single()
+        ConfiguredAggregateColumn.withDefault(this, defaultValue)
 
     public fun path(vararg names: String): ColumnPath = ColumnPath(names.asList())
 
-    public infix fun <C> ColumnSet<C>.into(name: String): ColumnSet<C> = ConfiguredAggregateColumn.withPath(this, pathOf(name))
+    public infix fun <C> ColumnSet<C>.into(name: String): ColumnSet<C> =
+        ConfiguredAggregateColumn.withPath(this, pathOf(name))
 
     public infix fun <C> SingleColumn<C>.into(name: String): SingleColumn<C> =
-        ConfiguredAggregateColumn.withPath(this, pathOf(name)).single()
+        ConfiguredAggregateColumn.withPath(this, pathOf(name))
 
-    public infix fun <C> ColumnSet<C>.into(path: ColumnPath): ColumnSet<C> = ConfiguredAggregateColumn.withPath(this, path)
+    public infix fun <C> ColumnSet<C>.into(path: ColumnPath): ColumnSet<C> =
+        ConfiguredAggregateColumn.withPath(this, path)
 
     public infix fun <C> SingleColumn<C>.into(path: ColumnPath): SingleColumn<C> =
-        ConfiguredAggregateColumn.withPath(this, path).single()
+        ConfiguredAggregateColumn.withPath(this, path)
 }

core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/annotations/ImportDataSchema.kt

@@ -49,9 +49,21 @@ public annotation class CsvOptions(
     public val delimiter: Char,
 )
 
+/**
+ * An annotation class that represents options for JDBC connection.
+ *
+ * @property [user] The username for the JDBC connection. Default value is an empty string.
+ * If [extractCredFromEnv] is true, the [user] value will be interpreted as key for system environment variable.
+ * @property [password] The password for the JDBC connection. Default value is an empty string.
+ * If [extractCredFromEnv] is true, the [password] value will be interpreted as key for system environment variable.
+ * @property [extractCredFromEnv] Whether to extract the JDBC credentials from environment variables. Default value is false.
+ * @property [tableName] The name of the table for the JDBC connection. Default value is an empty string.
+ * @property [sqlQuery] The SQL query to be executed in the JDBC connection. Default value is an empty string.
+ */
 public annotation class JdbcOptions(
-    public val user: String = "", // TODO: I'm not sure about the default parameters
-    public val password: String = "", // TODO: I'm not sure about the default parameters)
+    public val user: String = "",
+    public val password: String = "",
+    public val extractCredFromEnv: Boolean = false,
     public val tableName: String = "",
     public val sqlQuery: String = ""
 )