Skip to content

Commit 156a607

Browse files
ermolenkodevermolenkodev
authored
Merge pull request #290 from ermolenkodev/unified-tables-dev
Add support for swing tables integration in IDEA plugin
2 parents fbdc255 + 2b2fefa commit 156a607

File tree

9 files changed

+201
-82
lines changed

9 files changed

+201
-82
lines changed

‎core/build.gradle.kts

Lines changed: 11 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -52,26 +52,6 @@ kotlin.sourceSets {
5252
}
5353
}
5454

55-
tasks.lintKotlinMain {
56-
exclude("**/*keywords*/**")
57-
exclude {
58-
it.name.endsWith(".Generated.kt")
59-
}
60-
exclude {
61-
it.name.endsWith("\$Extensions.kt")
62-
}
63-
}
64-
65-
tasks.lintKotlinTest {
66-
exclude {
67-
it.name.endsWith(".Generated.kt")
68-
}
69-
exclude {
70-
it.name.endsWith("\$Extensions.kt")
71-
}
72-
enabled = true
73-
}
74-
7555
korro {
7656
docs = fileTree(rootProject.rootDir) {
7757
include("docs/StardustDocs/topics/*.md")
@@ -118,6 +98,17 @@ kotlinter {
11898
)
11999
}
120100

101+
tasks.withType<org.jmailen.gradle.kotlinter.tasks.LintTask> {
102+
exclude("**/*keywords*/**")
103+
exclude {
104+
it.name.endsWith(".Generated.kt")
105+
}
106+
exclude {
107+
it.name.endsWith("\$Extensions.kt")
108+
}
109+
enabled = true
110+
}
111+
121112
kotlin {
122113
explicitApi()
123114
}
@@ -128,7 +119,6 @@ tasks.withType<JavaCompile> {
128119
}
129120

130121
tasks.withType<org.jetbrains.kotlin.gradle.tasks.KotlinCompile> {
131-
dependsOn(tasks.lintKotlin)
132122
kotlinOptions {
133123
freeCompilerArgs = freeCompilerArgs + listOf("-Xinline-classes", "-Xopt-in=kotlin.RequiresOptIn")
134124
}

‎core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/DisableRowsLimitWrapper.kt

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
package org.jetbrains.kotlinx.dataframe.jupyter
2+
3+
import org.jetbrains.kotlinx.dataframe.AnyFrame
4+
5+
/**
6+
* Allows for disabling the rows limit when generating a DISPLAY output in Jupyter.
7+
*/
8+
public data class DisableRowsLimitWrapper(public val value: AnyFrame)

‎core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/Integration.kt

Lines changed: 45 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -6,29 +6,8 @@ import org.jetbrains.kotlinx.dataframe.AnyCol
66
import org.jetbrains.kotlinx.dataframe.AnyFrame
77
import org.jetbrains.kotlinx.dataframe.AnyRow
88
import org.jetbrains.kotlinx.dataframe.annotations.DataSchema
9-
import org.jetbrains.kotlinx.dataframe.api.Convert
10-
import org.jetbrains.kotlinx.dataframe.api.FormattedFrame
11-
import org.jetbrains.kotlinx.dataframe.api.Gather
12-
import org.jetbrains.kotlinx.dataframe.api.GroupBy
13-
import org.jetbrains.kotlinx.dataframe.api.Merge
14-
import org.jetbrains.kotlinx.dataframe.api.Pivot
15-
import org.jetbrains.kotlinx.dataframe.api.PivotGroupBy
16-
import org.jetbrains.kotlinx.dataframe.api.ReducedGroupBy
17-
import org.jetbrains.kotlinx.dataframe.api.ReducedPivot
18-
import org.jetbrains.kotlinx.dataframe.api.ReducedPivotGroupBy
19-
import org.jetbrains.kotlinx.dataframe.api.Split
20-
import org.jetbrains.kotlinx.dataframe.api.SplitWithTransform
21-
import org.jetbrains.kotlinx.dataframe.api.Update
22-
import org.jetbrains.kotlinx.dataframe.api.asColumnGroup
23-
import org.jetbrains.kotlinx.dataframe.api.asDataFrame
24-
import org.jetbrains.kotlinx.dataframe.api.columnsCount
25-
import org.jetbrains.kotlinx.dataframe.api.dataFrameOf
26-
import org.jetbrains.kotlinx.dataframe.api.frames
27-
import org.jetbrains.kotlinx.dataframe.api.into
28-
import org.jetbrains.kotlinx.dataframe.api.isColumnGroup
9+
import org.jetbrains.kotlinx.dataframe.api.*
2910
import org.jetbrains.kotlinx.dataframe.api.name
30-
import org.jetbrains.kotlinx.dataframe.api.toDataFrame
31-
import org.jetbrains.kotlinx.dataframe.api.values
3211
import org.jetbrains.kotlinx.dataframe.codeGen.CodeWithConverter
3312
import org.jetbrains.kotlinx.dataframe.columns.ColumnGroup
3413
import org.jetbrains.kotlinx.dataframe.columns.ColumnReference
@@ -111,42 +90,42 @@ internal class Integration(
11190
}
11291

11392
with(JupyterHtmlRenderer(config.display, this)) {
93+
render<DisableRowsLimitWrapper>(
94+
{ "DataRow: index = ${it.value.rowsCount()}, columnsCount = ${it.value.columnsCount()}" },
95+
applyRowsLimit = false
96+
)
97+
11498
render<HtmlData> { notebook.renderHtmlAsIFrameIfNeeded(it) }
11599
render<AnyRow>(
116-
{ it.toDataFrame() },
117100
{ "DataRow: index = ${it.index()}, columnsCount = ${it.columnsCount()}" },
118101
)
119102
render<ColumnGroup<*>>(
120-
{ it.asDataFrame() },
121103
{ """ColumnGroup: name = "${it.name}", rowsCount = ${it.rowsCount()}, columnsCount = ${it.columnsCount()}""" },
122104
)
123105
render<AnyCol>(
124-
{ dataFrameOf(it) },
125106
{ """DataColumn: name = "${it.name}", type = ${renderType(it.type())}, size = ${it.size()}""" },
126107
)
127108
render<AnyFrame>(
128-
{ it },
129-
{ "DataFrame: rowsCount = ${it.rowsCount()}, columnsCount = ${it.columnsCount()}" },
109+
{ "DataFrame: rowsCount = ${it.rowsCount()}, columnsCount = ${it.columnsCount()}" }
130110
)
131111
render<FormattedFrame<*>>(
132-
{ it.df },
133112
{ "DataFrame: rowsCount = ${it.df.rowsCount()}, columnsCount = ${it.df.columnsCount()}" },
134113
modifyConfig = { getDisplayConfiguration(it) },
135114
)
136-
render<GroupBy<*, *>>({ it.toDataFrame() }, { "GroupBy" })
137-
render<ReducedGroupBy<*, *>>({ it.into(it.groupBy.groups.name()) }, { "ReducedGroupBy" })
138-
render<Pivot<*>>({ it.frames().toDataFrame() }, { "Pivot" })
139-
render<ReducedPivot<*>>({ it.values().toDataFrame() }, { "ReducedPivot" })
140-
render<PivotGroupBy<*>>({ it.frames() }, { "PivotGroupBy" })
141-
render<ReducedPivotGroupBy<*>>({ it.values() }, { "ReducedPivotGroupBy" })
142-
render<SplitWithTransform<*, *, *>>({ it.into() }, { "Split" })
143-
render<Split<*, *>>({ it.toDataFrame() }, { "Split" })
144-
render<Merge<*, *, *>>({ it.into("merged") }, { "Merge" })
145-
render<Gather<*, *, *, *>>({ it.into("key", "value") }, { "Gather" })
115+
render<GroupBy<*, *>>({ "GroupBy" })
116+
render<ReducedGroupBy<*, *>>({ "ReducedGroupBy" })
117+
render<Pivot<*>>({ "Pivot" })
118+
render<ReducedPivot<*>>({ "ReducedPivot" })
119+
render<PivotGroupBy<*>>({ "PivotGroupBy" })
120+
render<ReducedPivotGroupBy<*>>({ "ReducedPivotGroupBy" })
121+
render<SplitWithTransform<*, *, *>>({ "Split" })
122+
render<Split<*, *>>({ "Split" })
123+
render<Merge<*, *, *>>({ "Merge" })
124+
render<Gather<*, *, *, *>>({ "Gather" })
146125
render<IMG> { HTML(it.toString()) }
147126
render<IFRAME> { HTML(it.toString()) }
148-
render<Update<*, *>>({ it.df }, { "Update" })
149-
render<Convert<*, *>>({ it.df }, { "Convert" })
127+
render<Update<*, *>>({ "Update" })
128+
render<Convert<*, *>>({ "Convert" })
150129
}
151130

152131
import("org.jetbrains.kotlinx.dataframe.api.*")
@@ -156,6 +135,7 @@ internal class Integration(
156135
import("org.jetbrains.kotlinx.dataframe.columns.*")
157136
import("org.jetbrains.kotlinx.dataframe.jupyter.ImportDataSchema")
158137
import("org.jetbrains.kotlinx.dataframe.jupyter.importDataSchema")
138+
import("org.jetbrains.kotlinx.dataframe.jupyter.KotlinNotebookPluginUtils")
159139
import("java.net.URL")
160140
import("java.io.File")
161141
import("kotlinx.datetime.Instant")
@@ -265,3 +245,28 @@ public fun KotlinKernelHost.useSchemas(schemaClasses: Iterable<KClass<*>>) {
265245
public fun KotlinKernelHost.useSchemas(vararg schemaClasses: KClass<*>): Unit = useSchemas(schemaClasses.asIterable())
266246

267247
public inline fun <reified T> KotlinKernelHost.useSchema(): Unit = useSchemas(T::class)
248+
249+
/**
250+
* Converts [dataframeLike] to [AnyFrame].
251+
* If [dataframeLike] is already [AnyFrame] then it is returned as is.
252+
* If it's not possible to convert [dataframeLike] to [AnyFrame] then [IllegalArgumentException] is thrown.
253+
*/
254+
internal fun convertToDataFrame(dataframeLike: Any): AnyFrame =
255+
when (dataframeLike) {
256+
is Pivot<*> -> dataframeLike.frames().toDataFrame()
257+
is ReducedPivot<*> -> dataframeLike.values().toDataFrame()
258+
is PivotGroupBy<*> -> dataframeLike.frames()
259+
is ReducedPivotGroupBy<*> -> dataframeLike.values()
260+
is SplitWithTransform<*, *, *> -> dataframeLike.into()
261+
is Merge<*, *, *> -> dataframeLike.into("merged")
262+
is Gather<*, *, *, *> -> dataframeLike.into("key", "value")
263+
is Update<*, *> -> dataframeLike.df
264+
is Convert<*, *> -> dataframeLike.df
265+
is FormattedFrame<*> -> dataframeLike.df
266+
is AnyCol -> dataFrameOf(dataframeLike)
267+
is AnyRow -> dataframeLike.toDataFrame()
268+
is GroupBy<*, *> -> dataframeLike.toDataFrame()
269+
is AnyFrame -> dataframeLike
270+
is DisableRowsLimitWrapper -> dataframeLike.value
271+
else -> throw IllegalArgumentException("Unsupported type: ${dataframeLike::class}")
272+
}

‎core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/JupyterHtmlRenderer.kt

Lines changed: 48 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,41 @@
11
package org.jetbrains.kotlinx.dataframe.jupyter
22

3-
import org.jetbrains.kotlinx.dataframe.DataFrame
4-
import org.jetbrains.kotlinx.dataframe.io.DisplayConfiguration
3+
import com.beust.klaxon.json
4+
import org.jetbrains.kotlinx.dataframe.api.rows
5+
import org.jetbrains.kotlinx.dataframe.api.toDataFrame
6+
import org.jetbrains.kotlinx.dataframe.io.*
57
import org.jetbrains.kotlinx.dataframe.io.initHtml
6-
import org.jetbrains.kotlinx.dataframe.io.toHTML
8+
import org.jetbrains.kotlinx.dataframe.nrow
9+
import org.jetbrains.kotlinx.dataframe.size
10+
import org.jetbrains.kotlinx.jupyter.api.*
711
import org.jetbrains.kotlinx.jupyter.api.libraries.JupyterIntegration
8-
import org.jetbrains.kotlinx.jupyter.api.renderHtmlAsIFrameIfNeeded
12+
13+
/** Starting from this version, dataframe integration will respond with additional data for rendering in Kotlin Notebooks plugin. */
14+
private const val MIN_KERNEL_VERSION_FOR_NEW_TABLES_UI = "0.11.0.311"
915

1016
internal class JupyterHtmlRenderer(
1117
val display: DisplayConfiguration,
1218
val builder: JupyterIntegration.Builder,
1319
)
1420

1521
internal inline fun <reified T : Any> JupyterHtmlRenderer.render(
16-
crossinline getDf: (T) -> DataFrame<*>,
1722
noinline getFooter: (T) -> String,
18-
crossinline modifyConfig: T.(DisplayConfiguration) -> DisplayConfiguration = { it }
23+
crossinline modifyConfig: T.(DisplayConfiguration) -> DisplayConfiguration = { it },
24+
applyRowsLimit: Boolean = true
1925
) = builder.renderWithHost<T> { host, value ->
2026
val contextRenderer = JupyterCellRenderer(this.notebook, host)
2127
val reifiedDisplayConfiguration = value.modifyConfig(display)
2228
val footer = getFooter(value)
23-
val html = getDf(value).toHTML(
29+
30+
val df = convertToDataFrame(value)
31+
32+
val limit = if (applyRowsLimit) {
33+
reifiedDisplayConfiguration.rowsLimit ?: df.nrow
34+
} else {
35+
df.nrow
36+
}
37+
38+
val html = df.toHTML(
2439
reifiedDisplayConfiguration,
2540
extraHtml = initHtml(
2641
includeJs = reifiedDisplayConfiguration.isolatedOutputs,
@@ -30,5 +45,30 @@ internal inline fun <reified T : Any> JupyterHtmlRenderer.render(
3045
contextRenderer
3146
) { footer }
3247

33-
notebook.renderHtmlAsIFrameIfNeeded(html)
48+
if (notebook.kernelVersion >= KotlinKernelVersion.from(MIN_KERNEL_VERSION_FOR_NEW_TABLES_UI)!!) {
49+
val jsonEncodedDf = json {
50+
obj(
51+
"nrow" to df.size.nrow,
52+
"ncol" to df.size.ncol,
53+
"columns" to df.columnNames(),
54+
"kotlin_dataframe" to encodeFrame(df.rows().take(limit).toDataFrame())
55+
)
56+
}.toJsonString()
57+
notebook.renderAsIFrameAsNeeded(html, jsonEncodedDf)
58+
} else {
59+
notebook.renderHtmlAsIFrameIfNeeded(html)
60+
}
61+
}
62+
63+
internal fun Notebook.renderAsIFrameAsNeeded(data: HtmlData, jsonEncodedDf: String): MimeTypedResult {
64+
val textHtml = if (jupyterClientType == JupyterClientType.KOTLIN_NOTEBOOK) {
65+
data.generateIframePlaneText(currentColorScheme)
66+
} else {
67+
data.toString(currentColorScheme)
68+
}
69+
70+
return mimeResult(
71+
"text/html" to textHtml,
72+
"application/kotlindataframe+json" to jsonEncodedDf
73+
).also { it.isolatedHtml = false }
3474
}

‎core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/KotlinNotebookPluginUtils.kt

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
package org.jetbrains.kotlinx.dataframe.jupyter
2+
3+
import org.jetbrains.kotlinx.dataframe.AnyFrame
4+
import org.jetbrains.kotlinx.dataframe.api.filter
5+
6+
/**
7+
* A class with utility methods for Kotlin Notebook Plugin integration.
8+
* Kotlin Notebook Plugin is acts as a client of Kotlin Jupyter kernel and use this functionality
9+
* for dynamic pagination when rendering dataframes.
10+
* The plugin sends Kotlin following code to the kernel to evaluate
11+
* DISPLAY(KotlinNotebooksPluginUtils.getRowsSubsetForRendering(Out[x], 0, 20), "")
12+
*/
13+
public object KotlinNotebookPluginUtils {
14+
/**
15+
* Returns a subset of rows from the given dataframe for rendering.
16+
* It's used for example for dynamic pagination in Kotlin Notebook Plugin.
17+
*/
18+
public fun getRowsSubsetForRendering(
19+
dataFrameLike: Any?,
20+
startIdx: Int,
21+
endIdx: Int
22+
): DisableRowsLimitWrapper = when (dataFrameLike) {
23+
null -> throw IllegalArgumentException("Dataframe is null")
24+
else -> getRowsSubsetForRendering(convertToDataFrame(dataFrameLike), startIdx, endIdx)
25+
}
26+
27+
/**
28+
* Returns a subset of rows from the given dataframe for rendering.
29+
* It's used for example for dynamic pagination in Kotlin Notebook Plugin.
30+
*/
31+
public fun getRowsSubsetForRendering(df: AnyFrame, startIdx: Int, endIdx: Int): DisableRowsLimitWrapper =
32+
DisableRowsLimitWrapper(df.filter { it.index() in startIdx until endIdx })
33+
}

‎core/src/test/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/RenderingTests.kt

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
package org.jetbrains.kotlinx.dataframe.jupyter
22

3+
import com.beust.klaxon.*
34
import io.kotest.matchers.shouldBe
45
import io.kotest.matchers.string.shouldContain
56
import io.kotest.matchers.string.shouldNotContain
67
import org.intellij.lang.annotations.Language
8+
import org.jetbrains.kotlinx.jupyter.api.MimeTypedResult
79
import org.jetbrains.kotlinx.jupyter.testkit.JupyterReplTestCase
810
import org.junit.Test
911

@@ -74,4 +76,53 @@ class RenderingTests : JupyterReplTestCase() {
7476
htmlLight shouldNotContain darkClassAttribute
7577
htmlDark shouldContain darkClassAttribute
7678
}
79+
80+
@Test
81+
fun `test kotlin notebook plugin utils rows subset`() {
82+
@Language("kts")
83+
val result = exec<MimeTypedResult>(
84+
"""
85+
data class Row(val id: Int)
86+
val df = (1..100).map { Row(it) }.toDataFrame()
87+
KotlinNotebookPluginUtils.getRowsSubsetForRendering(df, 20 , 50)
88+
""".trimIndent()
89+
)
90+
91+
val json = parseDataframeJson(result)
92+
93+
json.int("nrow") shouldBe 30
94+
json.int("ncol") shouldBe 1
95+
96+
val rows = json.array<JsonArray<*>>("kotlin_dataframe")!!
97+
rows.getObj(0).int("id") shouldBe 21
98+
rows.getObj(rows.lastIndex).int("id") shouldBe 50
99+
}
100+
101+
private fun parseDataframeJson(result: MimeTypedResult): JsonObject {
102+
val parser = Parser.default()
103+
return parser.parse(StringBuilder(result["application/kotlindataframe+json"]!!)) as JsonObject
104+
}
105+
106+
private fun JsonArray<*>.getObj(index: Int) = this.get(index) as JsonObject
107+
108+
@Test
109+
fun `test kotlin notebook plugin utils groupby`() {
110+
@Language("kts")
111+
val result = exec<MimeTypedResult>(
112+
"""
113+
data class Row(val id: Int, val group: Int)
114+
val df = (1..100).map { Row(it, if (it <= 50) 1 else 2) }.toDataFrame()
115+
KotlinNotebookPluginUtils.getRowsSubsetForRendering(df.groupBy("group"), 0, 10)
116+
""".trimIndent()
117+
)
118+
119+
val json = parseDataframeJson(result)
120+
121+
json.int("nrow") shouldBe 2
122+
json.int("ncol") shouldBe 2
123+
124+
val rows = json.array<JsonArray<*>>("kotlin_dataframe")!!
125+
rows.getObj(0).array<JsonObject>("group1")!!.size shouldBe 50
126+
rows.getObj(1).array<JsonObject>("group1")!!.size shouldBe 50
127+
}
77128
}

‎core/src/test/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/SampleNotebooksTests.kt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ class SampleNotebooksTests : DataFrameJupyterTest() {
3232
)
3333

3434
@Test
35+
@Ignore
3536
fun wine() = exampleTest(
3637
"wine", "WineNetWIthKotlinDL",
3738
replacer = CodeReplacer.byMap(

‎gradle/libs.versions.toml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
[versions]
2-
ksp = "1.8.10-1.0.9"
3-
kotlinJupyter = "0.11.0-198"
2+
ksp = "1.8.20-Beta-1.0.9"
3+
kotlinJupyter = "0.11.0-311"
44
ktlint = "3.4.5"
5-
kotlin = "1.8.10"
5+
kotlin = "1.8.20-Beta"
66
dokka = "1.8.10"
77
libsPublisher = "0.0.60-dev-30"
88
# "Bootstrap" version of the dataframe, used in the build itself to generate @DataSchema APIs,

0 commit comments

Comments
 (0)