Descriptors in libraries.json

docs/libraries.md

@@ -933,13 +933,20 @@ resources. This file should contain FQNs of all integration classes in the JSON
         {
             "fqn": "org.jetbrains.kotlinx.jupyter.example.GettingStartedIntegration"
         }
-    ]
+    ],
+    "descriptors": []
 }
 ```
 
 Classes derived from the `LibraryDefinition` interface should be added to the `definitions` array.
 Classes derived from the `LibraryDefinitionProducer` interface should be added to the `producers` array.
 
+Additionally, you're allowed to add full custom library descriptors to the `descriptors` array.
+This is an advanced option that allows you to load additional dependencies or add additional imports using the
+[library descriptor API](#creating-a-library-descriptor).
+This can be particularly useful for libraries that cannot depend on the jupyter-api artifact but still want to modify
+the kernel behavior when loaded.
+
 For more information, see:
 
 * [Libraries repository](https://github.com/Kotlin/kotlin-jupyter-libraries)

jupyter-lib/api/src/main/kotlin/org/jetbrains/kotlinx/jupyter/api/libraries/Scanning.kt

@@ -1,6 +1,7 @@
 package org.jetbrains.kotlinx.jupyter.api.libraries
 
 import kotlinx.serialization.Serializable
+import kotlinx.serialization.json.JsonObject
 import org.jetbrains.kotlinx.jupyter.api.TypeName
 
 /**
@@ -48,9 +49,13 @@ data class LibrariesProducerDeclaration(
 /**
  * Serialized form of this class instance is a correct content of
  * [KOTLIN_JUPYTER_LIBRARIES_FILE_NAME] file, and vice versa.
+ *
+ * @param descriptors List of library descriptor JSON objects adhering to
+ *   `org.jetbrains.kotlinx.jupyter.libraries.LibraryDescriptor`.
  */
 @Serializable
 data class LibrariesScanResult(
     val definitions: List<LibrariesDefinitionDeclaration> = emptyList(),
     val producers: List<LibrariesProducerDeclaration> = emptyList(),
+    val descriptors: List<JsonObject> = emptyList(),
 )

jupyter-lib/kotlin-jupyter-api-gradle-plugin/src/main/kotlin/org/jetbrains/kotlinx/jupyter/api/plugin/KotlinJupyterPluginExtension.kt

@@ -15,11 +15,13 @@ class KotlinJupyterPluginExtension(
 
     private val libraryProducers: MutableSet<FQNAware> = mutableSetOf()
     private val libraryDefinitions: MutableSet<FQNAware> = mutableSetOf()
+    private val libraryDescriptors: MutableSet<String> = mutableSetOf()
 
     internal val libraryFqns get() =
         LibrariesScanResult(
             definitions = libraryDefinitions,
             producers = libraryProducers,
+            descriptors = libraryDescriptors,
         )
 
     internal fun addDependenciesIfNeeded() {
@@ -42,6 +44,7 @@ class KotlinJupyterPluginExtension(
     /**
      * Add adding library integrations by specifying their fully qualified names
      */
+    @Suppress("unused")
     fun integrations(action: IntegrationsSpec.() -> Unit) {
         IntegrationsSpec().apply(action)
     }
@@ -56,6 +59,11 @@ class KotlinJupyterPluginExtension(
         fun definition(className: String) {
             libraryDefinitions.add(FQNAware(className))
         }
+
+        @Suppress("unused")
+        fun descriptor(descriptor: String) {
+            libraryDescriptors.add(descriptor)
+        }
     }
 
     companion object {

jupyter-lib/kotlin-jupyter-api-gradle-plugin/src/main/kotlin/org/jetbrains/kotlinx/jupyter/api/plugin/tasks/JupyterApiResourcesTask.kt

@@ -30,6 +30,13 @@ open class JupyterApiResourcesTask : DefaultTask() {
     @Input
     var libraryDefinitions: List<String> = emptyList()
 
+    /**
+     * List of JSON library descriptors in the
+     * [library descriptor format](https://github.com/Kotlin/kotlin-jupyter/blob/master/docs/libraries.md#creating-a-library-descriptor).
+     */
+    @Input
+    var libraryDescriptors: List<String> = emptyList()
+
     @OutputDirectory
     val outputDir: File = project.getBuildDirectory().resolve("jupyterProcessedResources")
 
@@ -42,6 +49,7 @@ open class JupyterApiResourcesTask : DefaultTask() {
             LibrariesScanResult(
                 definitions = libraryDefinitions.map { FQNAware(it) }.toSet(),
                 producers = libraryProducers.map { FQNAware(it) }.toSet(),
+                descriptors = libraryDescriptors.toSet(),
             )
 
         val resultObject =
@@ -70,14 +78,16 @@ open class JupyterApiResourcesTask : DefaultTask() {
         }
 
         return LibrariesScanResult(
-            fqns("definitions"),
-            fqns("producers"),
+            definitions = fqns("definitions"),
+            producers = fqns("producers"),
+            descriptors = emptySet(),
         )
     }
 
     operator fun LibrariesScanResult.plus(other: LibrariesScanResult): LibrariesScanResult =
         LibrariesScanResult(
-            definitions + other.definitions,
-            producers + other.producers,
+            definitions = definitions + other.definitions,
+            producers = producers + other.producers,
+            descriptors = descriptors + other.descriptors,
         )
 }

jupyter-lib/kotlin-jupyter-api-gradle-plugin/src/main/kotlin/org/jetbrains/kotlinx/jupyter/api/plugin/util/LibrariesUtil.kt

@@ -7,6 +7,7 @@ data class FQNAware(
 class LibrariesScanResult(
     val definitions: Set<FQNAware> = emptySet(),
     val producers: Set<FQNAware> = emptySet(),
+    val descriptors: Set<String> = emptySet(),
 )
 
 val emptyScanResult = LibrariesScanResult()

jupyter-lib/shared-compiler/src/main/kotlin/org/jetbrains/kotlinx/jupyter/libraries/LibrariesScanner.kt

@@ -1,6 +1,7 @@
 package org.jetbrains.kotlinx.jupyter.libraries
 
 import kotlinx.serialization.json.Json
+import kotlinx.serialization.json.JsonObject
 import org.jetbrains.kotlinx.jupyter.api.KotlinKernelHost
 import org.jetbrains.kotlinx.jupyter.api.LibraryLoader
 import org.jetbrains.kotlinx.jupyter.api.Notebook
@@ -27,6 +28,7 @@ class LibrariesScanner(
 
     private val processedFQNs = mutableSetOf<TypeName>()
     private val discardedFQNs = mutableSetOf<TypeName>()
+    private val processedDescriptors = mutableSetOf<JsonObject>()
 
     private fun <I : LibrariesInstantiable<*>> Iterable<I>.filterNamesToLoad(
         host: KotlinKernelHost,
@@ -46,10 +48,14 @@ class LibrariesScanner(
                     discardedFQNs.add(typeName)
                     false
                 }
+
                 null -> typeName !in discardedFQNs && processedFQNs.add(typeName)
             }
         }
 
+    private fun Iterable<JsonObject>.filterDescriptorsToLoad() =
+        filter { processedDescriptors.add(it) }
+
     fun addLibrariesFromClassLoader(
         classLoader: ClassLoader,
         host: KotlinKernelHost,
@@ -77,30 +83,36 @@ class LibrariesScanner(
         }
     }
 
+    private val jsonParser = Json { ignoreUnknownKeys = true }
+
     private fun scanForLibraries(
         classLoader: ClassLoader,
         host: KotlinKernelHost,
         integrationTypeNameRules: List<AcceptanceRule<TypeName>> = listOf(),
     ): LibrariesScanResult {
         val results =
-            classLoader.getResources("$KOTLIN_JUPYTER_RESOURCES_PATH/$KOTLIN_JUPYTER_LIBRARIES_FILE_NAME").toList().map { url ->
-                val contents = url.readText()
-                Json.decodeFromString<LibrariesScanResult>(contents)
-            }
+            classLoader.getResources("$KOTLIN_JUPYTER_RESOURCES_PATH/$KOTLIN_JUPYTER_LIBRARIES_FILE_NAME").toList()
+                .map { url ->
+                    val contents = url.readText()
+                    jsonParser.decodeFromString<LibrariesScanResult>(contents)
+                }
 
         val definitions = mutableListOf<LibrariesDefinitionDeclaration>()
         val producers = mutableListOf<LibrariesProducerDeclaration>()
+        val descriptors = mutableListOf<JsonObject>()
 
         for (result in results) {
             definitions.addAll(result.definitions)
             producers.addAll(result.producers)
+            descriptors.addAll(result.descriptors)
         }
 
         fun <I : LibrariesInstantiable<*>> Iterable<I>.filterNames() = filterNamesToLoad(host, integrationTypeNameRules)
 
         return LibrariesScanResult(
             definitions.filterNames(),
             producers.filterNames(),
+            descriptors.filterDescriptorsToLoad(),
         )
     }
 
@@ -140,6 +152,13 @@ class LibrariesScanner(
                 }
             }
         }
+
+        scanResult.descriptors.forEach {
+            val descriptor = parseLibraryDescriptor(it)
+            val definition = descriptor.convertToDefinition(arguments = emptyList())
+            definitions.add(definition)
+        }
+
         return definitions
     }
 

jupyter-lib/shared-compiler/src/main/kotlin/org/jetbrains/kotlinx/jupyter/libraries/Parsing.kt

@@ -2,16 +2,27 @@ package org.jetbrains.kotlinx.jupyter.libraries
 
 import kotlinx.serialization.SerializationException
 import kotlinx.serialization.json.Json
+import kotlinx.serialization.json.JsonObject
+import kotlinx.serialization.json.decodeFromJsonElement
 import org.jetbrains.kotlinx.jupyter.api.exceptions.ReplException
 import org.jetbrains.kotlinx.jupyter.protocol.api.KernelLoggerFactory
 
+private val jsonParser = Json { ignoreUnknownKeys = true }
+
 fun parseLibraryDescriptor(json: String): LibraryDescriptor =
     try {
-        Json.decodeFromString(json)
+        jsonParser.decodeFromString(json)
     } catch (e: SerializationException) {
         throw ReplException("Error during library deserialization. Library descriptor text:\n$json", e)
     }
 
+fun parseLibraryDescriptor(jsonObject: JsonObject): LibraryDescriptor =
+    try {
+        jsonParser.decodeFromJsonElement(jsonObject)
+    } catch (e: SerializationException) {
+        throw ReplException("Error during library deserialization. Library descriptor text:\n$jsonObject", e)
+    }
+
 fun parseLibraryDescriptors(
     loggerFactory: KernelLoggerFactory,
     libJsons: Map<String, String>,