Don't depend on http4k in shared-compiler module and refactor libraries processing modules

Author: ileasile

.gitignore

@@ -22,7 +22,7 @@ out/
 /teamcity-artifacts/
 
 # Folder with library descriptors
-libraries/
+/libraries/
 
 # Gradle caches and internal files
 .gradle/

build.gradle.kts

@@ -55,6 +55,11 @@ dependencies {
     // Clikt library for parsing output magics
     implementation(libs.clikt)
 
+    // Http4k for resolving library descriptors
+    implementation(libs.bundles.http4k) {
+        exclude(group = "org.jetbrains.kotlin")
+    }
+
     // Serialization implementation for kernel code
     implementation(libs.serialization.json)
 

jupyter-lib/shared-compiler/build.gradle.kts

@@ -12,9 +12,9 @@ repositories {
 
 dependencies {
     // Internal dependencies
-    api(projects.api)
-    api(projects.lib)
-    api(projects.commonDependencies)
+    api(projects.api) { isTransitive = false }
+    api(projects.lib) { isTransitive = false }
+    api(projects.commonDependencies) { isTransitive = false }
 
     // Standard dependencies
     compileOnly(libs.kotlin.stable.stdlib)
@@ -27,6 +27,9 @@ dependencies {
     compileOnly(libs.kotlin.dev.scriptingCompilerImplUnshaded)
     compileOnly(libs.kotlin.dev.scriptingDependencies) { isTransitive = false }
 
+    // Serialization runtime
+    compileOnly(libs.serialization.json)
+
     // Serialization compiler plugin (for notebooks, not for kernel code)
     compileOnly(libs.serialization.dev.unshadedPlugin)
 

jupyter-lib/shared-compiler/src/main/kotlin/org/jetbrains/kotlinx/jupyter/libraries/ByNothingLibraryResolutionInfo.kt

@@ -0,0 +1,6 @@
+package org.jetbrains.kotlinx.jupyter.libraries
+
+object ByNothingLibraryResolutionInfo : LibraryResolutionInfo {
+    override val key: String
+        get() = "nothing_"
+}

jupyter-lib/shared-compiler/src/main/kotlin/org/jetbrains/kotlinx/jupyter/libraries/ChainedLibraryResolver.kt

@@ -0,0 +1,24 @@
+package org.jetbrains.kotlinx.jupyter.libraries
+
+import org.jetbrains.kotlinx.jupyter.api.libraries.LibraryDefinition
+
+abstract class ChainedLibraryResolver(private val parent: LibraryResolver? = null) : LibraryResolver {
+    protected abstract fun tryResolve(reference: LibraryReference, arguments: List<Variable>): LibraryDefinition?
+    protected open fun save(reference: LibraryReference, definition: LibraryDefinition) {}
+    protected open fun shouldResolve(reference: LibraryReference): Boolean = true
+
+    override fun resolve(reference: LibraryReference, arguments: List<Variable>): LibraryDefinition? {
+        val shouldBeResolved = shouldResolve(reference)
+        if (shouldBeResolved) {
+            val result = tryResolve(reference, arguments)
+            if (result != null) return result
+        }
+
+        val parentResult = parent?.resolve(reference, arguments) ?: return null
+        if (shouldBeResolved) {
+            save(reference, parentResult)
+        }
+
+        return parentResult
+    }
+}

jupyter-lib/shared-compiler/src/main/kotlin/org/jetbrains/kotlinx/jupyter/libraries/LibrariesProcessor.kt

@@ -1,72 +1,7 @@
 package org.jetbrains.kotlinx.jupyter.libraries
 
-import org.jetbrains.kotlinx.jupyter.api.KotlinKernelVersion
-import org.jetbrains.kotlinx.jupyter.api.libraries.LibraryDefinition
 import org.jetbrains.kotlinx.jupyter.api.libraries.LibraryDefinitionProducer
-import org.jetbrains.kotlinx.jupyter.exceptions.ReplException
 
 interface LibrariesProcessor {
     fun processNewLibraries(arg: String): List<LibraryDefinitionProducer>
 }
-
-class LibrariesProcessorImpl(
-    private val libraries: LibraryResolver?,
-    private val kernelVersion: KotlinKernelVersion?,
-) : LibrariesProcessor {
-
-    /**
-     * Split a command argument into a set of library calls
-     * Need special processing of ',' to skip call argument delimiters in brackets
-     * E.g. "use lib1(3), lib2(2, 5)" should split into "lib1(3)" and "lib(2, 5)", not into "lib1(3)", "lib(2", "5)"
-     */
-    private fun splitLibraryCalls(text: String): List<String> {
-        var i = 0
-        var prev = 0
-        var commaDepth = 0
-        val result = mutableListOf<String>()
-        val delimiters = charArrayOf(',', '(', ')')
-        while (true) {
-            i = text.indexOfAny(delimiters, i)
-            if (i == -1) {
-                val res = text.substring(prev, text.length).trim()
-                if (res.isNotEmpty()) {
-                    result.add(res)
-                }
-                return result
-            }
-            when (text[i]) {
-                ',' -> if (commaDepth == 0) {
-                    val res = text.substring(prev, i).trim()
-                    if (res.isNotEmpty()) {
-                        result.add(res)
-                    }
-                    prev = i + 1
-                }
-                '(' -> commaDepth++
-                ')' -> commaDepth--
-            }
-            i++
-        }
-    }
-
-    private fun checkKernelVersionRequirements(name: String, library: LibraryDefinition) {
-        library.minKernelVersion?.let { minVersion ->
-            kernelVersion?.let { currentVersion ->
-                if (currentVersion < minVersion) {
-                    throw ReplException("Library '$name' requires at least $minVersion version of kernel. Current kernel version is $currentVersion. Please update kernel")
-                }
-            }
-        }
-    }
-
-    override fun processNewLibraries(arg: String): List<LibraryDefinitionProducer> =
-        splitLibraryCalls(arg).map {
-            val (libRef, vars) = parseReferenceWithArgs(it)
-            val library = libraries?.resolve(libRef, vars)
-                ?: throw ReplException("Unknown library '$libRef'")
-
-            checkKernelVersionRequirements(libRef.toString(), library)
-
-            TrivialLibraryDefinitionProducer(library)
-        }
-}

jupyter-lib/shared-compiler/src/main/kotlin/org/jetbrains/kotlinx/jupyter/libraries/LibraryResolutionInfo.kt

@@ -1,85 +1,5 @@
 package org.jetbrains.kotlinx.jupyter.libraries
 
-import java.io.File
-import java.net.URL
-import java.util.concurrent.ConcurrentHashMap
-
-abstract class LibraryResolutionInfo(
-    private val typeKey: String
-) : LibraryCacheable {
-    class ByNothing : LibraryResolutionInfo("nothing") {
-        override val args: List<Variable> = listOf()
-    }
-
-    class ByURL(val url: URL) : LibraryResolutionInfo("url") {
-        override val args = listOf(Variable("url", url.toString()))
-        override val shouldBeCachedLocally get() = false
-    }
-
-    class ByFile(val file: File) : LibraryResolutionInfo("file") {
-        override val args = listOf(Variable("file", file.path))
-        override val shouldBeCachedLocally get() = false
-    }
-
-    class ByDir(val librariesDir: File) : LibraryResolutionInfo("bundled") {
-        override val args = listOf(Variable("dir", librariesDir.path))
-        override val shouldBeCachedLocally get() = false
-    }
-
-    class ByGitRef(private val ref: String) : LibraryResolutionInfo("ref") {
-        override val valueKey: String
-            get() = sha
-
-        val sha: String by lazy {
-            val (resolvedSha, _) = KERNEL_LIBRARIES.getLatestCommitToLibraries(ref, null) ?: return@lazy ref
-            resolvedSha
-        }
-
-        override val args = listOf(Variable("ref", ref))
-    }
-
-    class Default(val string: String = "") : LibraryResolutionInfo("default") {
-        override val args: List<Variable> = listOf()
-        override val shouldBeCachedLocally get() = false
-    }
-
-    protected abstract val args: List<Variable>
-    protected open val valueKey: String
-        get() = args.joinToString { it.value }
-
-    val key: String by lazy { "${typeKey}_${replaceForbiddenChars(valueKey)}" }
-
-    override fun hashCode(): Int {
-        return key.hashCode()
-    }
-
-    override fun equals(other: Any?): Boolean {
-        if (this === other) return true
-        if (javaClass != other?.javaClass) return false
-
-        other as LibraryResolutionInfo
-
-        return valueKey == other.valueKey
-    }
-
-    override fun toString(): String {
-        return typeKey +
-            when {
-                args.isEmpty() -> ""
-                args.size == 1 -> "[${args[0].value}]"
-                else -> args.joinToString(", ", "[", "]") { "${it.name}=${it.value}" }
-            }
-    }
-
-    companion object {
-        private val gitRefsCache = ConcurrentHashMap<String, ByGitRef>()
-
-        fun getInfoByRef(ref: String): ByGitRef {
-            return gitRefsCache.getOrPut(ref, { ByGitRef(ref) })
-        }
-
-        fun replaceForbiddenChars(string: String): String {
-            return string.replace("""[<>/\\:"|?*]""".toRegex(), "_")
-        }
-    }
+interface LibraryResolutionInfo : LibraryCacheable {
+    val key: String
 }

jupyter-lib/shared-compiler/src/main/kotlin/org/jetbrains/kotlinx/jupyter/libraries/LibraryResolver.kt

@@ -0,0 +1,7 @@
+package org.jetbrains.kotlinx.jupyter.libraries
+
+import org.jetbrains.kotlinx.jupyter.api.libraries.LibraryDefinition
+
+interface LibraryResolver {
+    fun resolve(reference: LibraryReference, arguments: List<Variable>): LibraryDefinition?
+}

jupyter-lib/shared-compiler/src/main/kotlin/org/jetbrains/kotlinx/jupyter/libraries/ResolutionInfoProvider.kt

@@ -1,62 +1,7 @@
 package org.jetbrains.kotlinx.jupyter.libraries
 
-import org.jetbrains.kotlinx.jupyter.common.getHttp
-import java.io.File
-import java.net.URL
-
 interface ResolutionInfoProvider {
     var fallback: LibraryResolutionInfo
 
     fun get(string: String): LibraryResolutionInfo
-
-    companion object {
-        fun withDefaultDirectoryResolution(dir: File) = StandardResolutionInfoProvider(
-            LibraryResolutionInfo.ByDir(dir)
-        )
-
-        // Used in Kotlin Jupyter plugin for IDEA
-        @Suppress("unused")
-        fun withDefaultGitRefResolution(ref: String) = StandardResolutionInfoProvider(
-            LibraryResolutionInfo.getInfoByRef(ref)
-        )
-    }
-}
-
-object EmptyResolutionInfoProvider : ResolutionInfoProvider {
-    private val fallbackInfo = LibraryResolutionInfo.ByNothing()
-
-    override var fallback: LibraryResolutionInfo
-        get() = fallbackInfo
-        set(_) {}
-
-    override fun get(string: String): LibraryResolutionInfo {
-        if (string.isEmpty()) return fallback
-        return LibraryResolutionInfo.getInfoByRef(string)
-    }
-}
-
-class StandardResolutionInfoProvider(override var fallback: LibraryResolutionInfo) : ResolutionInfoProvider {
-    override fun get(string: String): LibraryResolutionInfo {
-        if (string.isEmpty()) return fallback
-        return tryGetAsRef(string) ?: tryGetAsDir(string) ?: tryGetAsFile(string) ?: tryGetAsURL(string) ?: fallback
-    }
-
-    private fun tryGetAsRef(ref: String): LibraryResolutionInfo? {
-        return if (KERNEL_LIBRARIES.checkRefExistence(ref)) LibraryResolutionInfo.getInfoByRef(ref) else null
-    }
-
-    private fun tryGetAsDir(dirName: String): LibraryResolutionInfo? {
-        val file = File(dirName)
-        return if (file.isDirectory) LibraryResolutionInfo.ByDir(file) else null
-    }
-
-    private fun tryGetAsFile(fileName: String): LibraryResolutionInfo? {
-        val file = File(fileName)
-        return if (file.isFile) LibraryResolutionInfo.ByFile(file) else null
-    }
-
-    private fun tryGetAsURL(url: String): LibraryResolutionInfo? {
-        val response = getHttp(url)
-        return if (response.status.successful) LibraryResolutionInfo.ByURL(URL(url)) else null
-    }
 }

jupyter-lib/shared-compiler/src/main/kotlin/org/jetbrains/kotlinx/jupyter/libraries/ResolutionInfoSwitcher.kt

@@ -0,0 +1,21 @@
+package org.jetbrains.kotlinx.jupyter.libraries
+
+class ResolutionInfoSwitcher<T>(private val infoProvider: ResolutionInfoProvider, initialSwitchVal: T, private val switcher: (T) -> LibraryResolutionInfo) {
+    private val defaultInfoCache = hashMapOf<T, LibraryResolutionInfo>()
+
+    var switch: T = initialSwitchVal
+        set(value) {
+            infoProvider.fallback = defaultInfoCache.getOrPut(value) { switcher(value) }
+            field = value
+        }
+
+    companion object {
+        // Used in Kotlin Jupyter plugin for IDEA
+        @Suppress("unused")
+        fun noop(provider: ResolutionInfoProvider): ResolutionInfoSwitcher<DefaultInfoSwitch> {
+            return ResolutionInfoSwitcher(provider, DefaultInfoSwitch.GIT_REFERENCE) {
+                provider.fallback
+            }
+        }
+    }
+}