Merge branch 'master' into unsupported-data-sources-examples

Jolanrensen · Jolanrensen · commit 3194e30a0274 · 2025-06-12T20:27:05.000+02:00
# Conflicts:
#	settings.gradle.kts
diff --git a/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/html.kt b/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/html.kt
@@ -77,7 +77,14 @@ internal val formatter = DataFrameFormatter(
 internal fun getResources(vararg resource: String) = resource.joinToString(separator = "\n") { getResourceText(it) }
 
 internal fun getResourceText(resource: String, vararg replacement: Pair<String, Any>): String {
-    val res = DataFrame::class.java.getResourceAsStream(resource) ?: error("Resource '$resource' not found")
+    /**
+     * The choice of loader is crucial here: it should always be a class loaded by the same class loader as the resource we load.
+     * I.e. [DataFrame] isn't a good fit because it might be loaded by Kotlin IDEA plugin (because Kotlin plugin
+     * loads DataFrame compiler plugin), and plugin's classloader knows nothing about the resources.
+     */
+    val loader = HtmlContent::class.java
+    val res = loader.getResourceAsStream(resource)
+        ?: error("Resource '$resource' not found. Load was attempted by $loader, loaded by ${loader.classLoader}")
     var template = InputStreamReader(res).readText()
     replacement.forEach {
         template = template.replace(it.first, it.second.toString())
diff --git a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/html.kt b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/html.kt
@@ -77,7 +77,14 @@ internal val formatter = DataFrameFormatter(
 internal fun getResources(vararg resource: String) = resource.joinToString(separator = "\n") { getResourceText(it) }
 
 internal fun getResourceText(resource: String, vararg replacement: Pair<String, Any>): String {
-    val res = DataFrame::class.java.getResourceAsStream(resource) ?: error("Resource '$resource' not found")
+    /**
+     * The choice of loader is crucial here: it should always be a class loaded by the same class loader as the resource we load.
+     * I.e. [DataFrame] isn't a good fit because it might be loaded by Kotlin IDEA plugin (because Kotlin plugin
+     * loads DataFrame compiler plugin), and plugin's classloader knows nothing about the resources.
+     */
+    val loader = HtmlContent::class.java
+    val res = loader.getResourceAsStream(resource)
+        ?: error("Resource '$resource' not found. Load was attempted by $loader, loaded by ${loader.classLoader}")
     var template = InputStreamReader(res).readText()
     replacement.forEach {
         template = template.replace(it.first, it.second.toString())
diff --git a/docs/StardustDocs/topics/Compiler-Plugin.md b/docs/StardustDocs/topics/Compiler-Plugin.md
@@ -1,9 +1,34 @@
 # Kotlin DataFrame Compiler Plugin
 
-Kotlin DataFrame compiler plugin: available in Gradle projects, is coming to Kotlin Notebook and Maven projects soon.
+<web-summary>
+Explore the Kotlin DataFrame Compiler Plugin —  
+a powerful tool providing on-the-fly type-safe column-accessors for dataframes.
+</web-summary>
 
-Check out this video that shows how expressions update the schema of a dataframe: 
+<card-summary>
+Explore the Kotlin DataFrame Compiler Plugin —  
+a powerful tool providing on-the-fly type-safe column-accessors for dataframes.
+</card-summary>
+
+<link-summary>
+Explore the Kotlin DataFrame Compiler Plugin —  
+a powerful tool providing on-the-fly type-safe column-accessors for dataframes.
+</link-summary>
+
+
+> Now available in Gradle projects, is coming soon to Kotlin Notebook and Maven projects.
+
+**Kotlin DataFrame Compiler Plugin** is a Kotlin compiler plugin that automatically generates  
+**[type-safe extension properties](extensionPropertiesApi.md)** for your dataframes,  
+allowing you to access columns and row values in a type-safe way and avoid mistakes in column names.
 
+## Why use it?
+
+- Access columns as regular properties: `df.name` instead of `df["name"]`.
+- Get full IDE and compiler support: autocompletion, refactoring, and type checking.
+- Improve code readability and safety when working with DataFrame.
+
+Check out this video that shows how expressions update the schema of a dataframe: 
 
 <video src="compiler_plugin.mp4" controls=""/>
 
@@ -113,3 +138,9 @@ fun main() {
 ```
 
 [Learn more](dataSchema.md) about data schema declarations
+
+## Examples
+
+* [Kotlin DataFrame in the IntelliJ IDEA project example](https://github.com/Kotlin/dataframe/blob/master/examples/kotlin-dataframe-plugin-example)  
+  — an IntelliJ IDEA project showcasing simple DataFrame expressions using the Compiler Plugin.
+* [](compilerPluginExamples.md) — few examples of Compiler Plugin usages.
diff --git a/docs/StardustDocs/topics/Home.topic b/docs/StardustDocs/topics/Home.topic
@@ -28,6 +28,7 @@
         <secondary>
             <title>Featured topics</title>
             <a href="Kotlin-DataFrame-Features-in-Kotlin-Notebook.md"/>
+            <a href="Compiler-Plugin.md"/>
             <a href="readSqlDatabases.md"/>
         </secondary>
 
diff --git a/docs/StardustDocs/topics/compilerPluginExamples.md b/docs/StardustDocs/topics/compilerPluginExamples.md
@@ -3,6 +3,9 @@
 This page provides a few examples that you can copy directly to your project.
 [Schema info](staticInterpretation.md#schema-info) will be a convenient way to observe the result of different operations.
 
+> See also an [IntelliJ IDEA project example](https://github.com/Kotlin/dataframe/blob/master/examples/kotlin-dataframe-plugin-example), 
+> showcasing simple DataFrame expressions using the Compiler Plugin.
+
 ### Example 1
 
 ```kotlin
diff --git a/docs/StardustDocs/topics/guides/Guides-And-Examples.md b/docs/StardustDocs/topics/guides/Guides-And-Examples.md
@@ -56,6 +56,11 @@ Explore our structured, in-depth guides to steadily improve your Kotlin DataFram
 
 Explore our extensive collection of practical examples and real-world analytics workflows.
 
+* [Kotlin DataFrame Compiler Plugin Example](https://github.com/Kotlin/dataframe/blob/master/examples/kotlin-dataframe-plugin-example)  
+  — a simple project demonstrating the usage of the [compiler plugin](Compiler-Plugin.md),  
+  showcasing DataFrame expressions with [extension properties](extensionPropertiesApi.md)  
+  that are generated on-the-fly in the IDEA project.
+
 * [Titanic Example](https://github.com/Kotlin/dataframe/blob/master/examples/notebooks/titanic/Titanic.ipynb)
   — discover the famous "Titanic"
   dataset with the Kotlin DataFrame analysis toolkit
diff --git a/examples/README.md b/examples/README.md
@@ -1,6 +1,8 @@
 # Examples of Kotlin Dataframe
 
 ### Idea examples
+* [plugin example](kotlin-dataframe-plugin-example) IDEA project with a 
+[Kotlin DataFrame Compiler Plugin](https://kotlin.github.io/dataframe/compiler-plugin.html) example.
 * [movies](idea-examples/movies) Using extension properties [Access API](https://kotlin.github.io/dataframe/apilevels.html) to perform a data cleaning task
 * [titanic](idea-examples/titanic)
 * [youtube](idea-examples/youtube)
diff --git a/examples/kotlin-dataframe-plugin-example/README.md b/examples/kotlin-dataframe-plugin-example/README.md
@@ -0,0 +1,10 @@
+# Kotlin DataFrame Compiler Plugin Example
+
+An IntelliJ IDEA project demonstrating the use of the  
+[Kotlin DataFrame Compiler Plugin](https://kotlin.github.io/dataframe/compiler-plugin.html).
+
+> **Note:** This project uses **dev versions** of the Kotlin and the Kotlin DataFrame plugin.  
+> See [build.gradle.kts](build.gradle.kts) for details.
+
+For proper functionality in IntelliJ IDEA requires version 2025.2 or higher 
+(Currently available only in [IntelliJ IDEA EAP](https://www.jetbrains.com/idea/nextversion/?_gl=1*1ojxffu*_gcl_au*MTk5NzUwODYzOS4xNzQ2NzkxMDMz*_ga*MTE0ODQ1MzY3OS4xNzM4OTY1NzM3*_ga_9J976DJZ68*czE3NDkyMTM4MzkkbzE5OCRnMSR0MTc0OTIxMzg0MSRqNTgkbDAkaDA.)).
diff --git a/examples/kotlin-dataframe-plugin-example/build.gradle.kts b/examples/kotlin-dataframe-plugin-example/build.gradle.kts
@@ -0,0 +1,32 @@
+import org.jlleitschuh.gradle.ktlint.KtlintExtension
+
+plugins {
+    id("org.jlleitschuh.gradle.ktlint") version "12.3.0"
+    kotlin("jvm") version "2.2.20-dev-3524"
+    kotlin("plugin.dataframe") version "2.2.20-dev-3524"
+}
+
+group = "org.example"
+version = "1.0-SNAPSHOT"
+
+repositories {
+    maven("https://packages.jetbrains.team/maven/p/kt/dev/")
+    mavenCentral()
+}
+
+dependencies {
+    implementation("org.jetbrains.kotlinx:dataframe:1.0.0-Beta2")
+    testImplementation(kotlin("test"))
+}
+
+tasks.test {
+    useJUnitPlatform()
+}
+kotlin {
+    jvmToolchain(11)
+}
+
+configure<KtlintExtension> {
+    version = "1.6.0"
+    // rules are set up through .editorconfig
+}
diff --git a/examples/kotlin-dataframe-plugin-example/gradle.properties b/examples/kotlin-dataframe-plugin-example/gradle.properties
@@ -0,0 +1,4 @@
+kotlin.code.style=official
+# Disabling incremental compilation will no longer be necessary
+# when https://youtrack.jetbrains.com/issue/KT-66735 is resolved.
+kotlin.incremental=false
diff --git a/examples/kotlin-dataframe-plugin-example/settings.gradle.kts b/examples/kotlin-dataframe-plugin-example/settings.gradle.kts
@@ -0,0 +1,11 @@
+pluginManagement {
+    repositories {
+        maven("https://packages.jetbrains.team/maven/p/kt/dev/")
+        mavenCentral()
+        gradlePluginPortal()
+    }
+}
+plugins {
+    id("org.gradle.toolchains.foojay-resolver-convention") version "0.8.0"
+}
+rootProject.name = "kotlin-dataframe-plugin-example"
diff --git a/examples/kotlin-dataframe-plugin-example/src/main/kotlin/org/jetbrains/kotlinx/dataframe/examples/plugin/Main.kt b/examples/kotlin-dataframe-plugin-example/src/main/kotlin/org/jetbrains/kotlinx/dataframe/examples/plugin/Main.kt
@@ -0,0 +1,93 @@
+package org.jetbrains.kotlinx.dataframe.examples.plugin
+
+import org.jetbrains.kotlinx.dataframe.DataFrame
+import org.jetbrains.kotlinx.dataframe.annotations.DataSchema
+import org.jetbrains.kotlinx.dataframe.api.add
+import org.jetbrains.kotlinx.dataframe.api.convert
+import org.jetbrains.kotlinx.dataframe.api.convertTo
+import org.jetbrains.kotlinx.dataframe.api.filter
+import org.jetbrains.kotlinx.dataframe.api.into
+import org.jetbrains.kotlinx.dataframe.api.rename
+import org.jetbrains.kotlinx.dataframe.api.renameToCamelCase
+import org.jetbrains.kotlinx.dataframe.api.with
+import org.jetbrains.kotlinx.dataframe.io.readCsv
+import org.jetbrains.kotlinx.dataframe.io.writeCsv
+import java.net.URL
+
+// Declare data schema for the DataFrame from jetbrains_repositories.csv.
+@DataSchema
+data class Repositories(
+    val full_name: String,
+    val html_url: URL,
+    val stargazers_count: Int,
+    val topics: String,
+    val watchers: Int,
+)
+
+// Define kinds of repositories.
+enum class RepoKind {
+    Kotlin,
+    IntelliJ,
+    Other,
+}
+
+// A rule for determining the kind of repository based on its name and topics.
+fun getKind(fullName: String, topics: List<String>): RepoKind {
+    fun checkContains(name: String) = name in topics || fullName.lowercase().contains(name)
+
+    return when {
+        checkContains("kotlin") -> RepoKind.Kotlin
+        checkContains("idea") || checkContains("intellij") -> RepoKind.IntelliJ
+        else -> RepoKind.Other
+    }
+}
+
+fun main() {
+    val repos = DataFrame
+        // Read DataFrame from the CSV file.
+        .readCsv("https://raw.githubusercontent.com/Kotlin/dataframe/master/data/jetbrains_repositories.csv")
+        // And convert it to match the `Repositories` schema.
+        .convertTo<Repositories>()
+
+    // With Compiler Plugin, the DataFrame schema changes immediately after each operation:
+    // For example, if a new column is added or the old one is renamed (or its type is changed)
+    // during the operation, you can use the new name immediately in the following operations:
+    repos
+        // Add a new "name" column...
+        .add("name") { full_name.substringAfterLast("/") }
+        // ... and now we can use "name" extension in DataFrame operations, such as `filter`.
+        .filter { name.lowercase().contains("kotlin") }
+
+    // Let's update the DataFrame with some operations using these features.
+    val reposUpdated = repos
+        // Rename columns to CamelCase.
+        // Note that after that, in the following operations, extension properties will have
+        // new names corresponding to the column names.
+        .renameToCamelCase()
+        // Rename "stargazersCount" column to "stars".
+        .rename { stargazersCount }.into("stars")
+        // And we can immediately use the updated name in the filtering.
+        .filter { stars > 50 }
+        // Convert values in the "topic" column (which were `String` initially)
+        // to the list of topics.
+        .convert { topics }.with {
+            val inner = it.removeSurrounding("[", "]")
+            if (inner.isEmpty()) emptyList() else inner.split(',').map(String::trim)
+        }
+        // Now "topics" is a `List<String>` column.
+        // Add a new column with the number of topics.
+        .add("topicCount") { topics.size }
+        // Add a new column with the kind of repository.
+        .add("kind") { getKind(fullName, topics) }
+
+    // Write the updated DataFrame to a CSV file.
+    reposUpdated.writeCsv("jetbrains_repositories_new.csv")
+
+    // TODO: Add Kandy Plot
+    //  reposUpdated.groupBy { kind }.max { stargazersCount  }.plot {
+    //      bars {
+    //          x(kind)
+    //          y(stargazersCount)
+    //      }
+    //  }
+}
diff --git a/settings.gradle.kts b/settings.gradle.kts
@@ -11,6 +11,7 @@ include("plugins:dataframe-gradle-plugin")
 include("plugins:symbol-processor")
 include("plugins:expressions-converter")
 include("plugins:kotlin-dataframe")
+include("plugins:public-api-modifier")
 include("tests")
 include("dataframe-json")
 include("dataframe-arrow")
@@ -19,13 +20,17 @@ include("dataframe-excel")
 include("dataframe-jdbc")
 include("dataframe-csv")
 include("dataframe-jupyter")
+include("dataframe-geo")
+include("dataframe-openapi-generator")
 include("core")
+include("dataframe-compiler-plugin-core")
 
 include("examples:idea-examples:titanic")
 include("examples:idea-examples:movies")
 include("examples:idea-examples:youtube")
 include("examples:idea-examples:json")
 include("examples:idea-examples:unsupported-data-sources")
+includeBuild("examples/kotlin-dataframe-plugin-example")
 
 val jupyterApiTCRepo: String by settings
 
@@ -45,9 +50,3 @@ pluginManagement {
 plugins {
     id("org.gradle.toolchains.foojay-resolver-convention") version "0.5.0"
 }
-include("dataframe-excel")
-include("core")
-include("dataframe-openapi-generator")
-include("dataframe-geo")
-include("plugins:public-api-modifier")
-include("dataframe-compiler-plugin-core")
