Skip to content

Commit eee13a0

Browse files
ileasileileasile
committed
Move library resolution logic to a common module and encapsulate it
1 parent eeb9b50 commit eee13a0

File tree

17 files changed

+159
-107
lines changed

17 files changed

+159
-107
lines changed

‎jupyter-lib/common-dependencies/src/main/kotlin/org/jetbrains/kotlinx/jupyter/common/libraries.kt

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
package org.jetbrains.kotlinx.jupyter.common
2+
3+
import kotlinx.serialization.json.JsonObject
4+
import kotlinx.serialization.json.JsonPrimitive
5+
import org.slf4j.Logger
6+
import org.slf4j.LoggerFactory
7+
import java.io.File
8+
9+
fun interface ExceptionsHandler {
10+
fun handle(logger: Logger, message: String, exception: Throwable)
11+
12+
object DEFAULT : ExceptionsHandler {
13+
override fun handle(logger: Logger, message: String, exception: Throwable) {
14+
logger.error(message)
15+
throw exception
16+
}
17+
}
18+
}
19+
20+
class LibraryDescriptorsManager private constructor(
21+
user: String,
22+
repo: String,
23+
private val remotePath: String,
24+
localPath: String,
25+
private val homePath: String,
26+
userPath: String,
27+
private val exceptionsHandler: ExceptionsHandler,
28+
userSettingsDir: File,
29+
private val logger: Logger,
30+
) {
31+
private val apiPrefix = "https://$GITHUB_API_HOST/repos/$user/$repo"
32+
val userLibrariesDir = userSettingsDir.resolve(userPath)
33+
val userCacheDir = userSettingsDir.resolve("cache")
34+
val localLibrariesDir = File(localPath)
35+
fun homeLibrariesDir(homeDir: File? = null) = (homeDir ?: File("")).resolve(homePath)
36+
37+
val localPropertiesFile = localLibrariesDir.resolve(PROPERTIES_FILE)
38+
39+
fun descriptorFileName(name: String) = "$name.$DESCRIPTOR_EXTENSION"
40+
41+
fun isLibraryDescriptor(file: File): Boolean {
42+
return file.isFile && file.name.endsWith(".$DESCRIPTOR_EXTENSION")
43+
}
44+
45+
fun getLatestCommitToLibraries(ref: String, sinceTimestamp: String?): Pair<String, String>? {
46+
return catchAll {
47+
var url = "$apiPrefix/commits?path=$remotePath&sha=$ref"
48+
if (sinceTimestamp != null) {
49+
url += "&since=$sinceTimestamp"
50+
}
51+
logger.info("Checking for new commits to library descriptors at $url")
52+
val arr = getHttp(url).jsonArray
53+
if (arr.isEmpty()) {
54+
if (sinceTimestamp != null) {
55+
getLatestCommitToLibraries(ref, null)
56+
} else {
57+
logger.info("Didn't find any commits to libraries at $url")
58+
null
59+
}
60+
} else {
61+
val commit = arr[0] as JsonObject
62+
val sha = (commit["sha"] as JsonPrimitive).content
63+
val timestamp = (((commit["commit"] as JsonObject)["committer"] as JsonObject)["date"] as JsonPrimitive).content
64+
sha to timestamp
65+
}
66+
}
67+
}
68+
69+
fun downloadLibraryDescriptor(ref: String, name: String): String {
70+
val url = "$apiPrefix/contents/$remotePath/$name.$DESCRIPTOR_EXTENSION?ref=$ref"
71+
logger.info("Requesting library descriptor at $url")
72+
val response = getHttp(url).jsonObject
73+
74+
val downloadURL = (response["download_url"] as JsonPrimitive).content
75+
val res = getHttp(downloadURL)
76+
return res.text
77+
}
78+
79+
fun checkRefExistence(ref: String): Boolean {
80+
val response = getHttp("$apiPrefix/contents/$remotePath?ref=$ref")
81+
return response.status.successful
82+
}
83+
84+
private fun <T> catchAll(message: String = "", body: () -> T): T? = try {
85+
body()
86+
} catch (e: Throwable) {
87+
exceptionsHandler.handle(logger, message, e)
88+
null
89+
}
90+
91+
companion object {
92+
private const val GITHUB_API_HOST = "api.github.com"
93+
private const val DESCRIPTOR_EXTENSION = "json"
94+
private const val PROPERTIES_FILE = ".properties"
95+
96+
fun getInstance(
97+
logger: Logger = LoggerFactory.getLogger(LibraryDescriptorsManager::class.java),
98+
exceptionsHandler: ExceptionsHandler = ExceptionsHandler.DEFAULT,
99+
): LibraryDescriptorsManager {
100+
return LibraryDescriptorsManager(
101+
"Kotlin",
102+
"kotlin-jupyter-libraries",
103+
"",
104+
"libraries",
105+
"libraries",
106+
"libraries",
107+
exceptionsHandler,
108+
File(System.getProperty("user.home")).resolve(".jupyter_kotlin"),
109+
logger
110+
)
111+
}
112+
}
113+
}

‎jupyter-lib/shared-compiler/src/main/kotlin/org/jetbrains/kotlinx/jupyter/libraries/LibraryResolutionInfo.kt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ abstract class LibraryResolutionInfo(
3131
get() = sha
3232

3333
val sha: String by lazy {
34-
val (resolvedSha, _) = getLatestCommitToLibraries(ref, null) ?: return@lazy ref
34+
val (resolvedSha, _) = KERNEL_LIBRARIES.getLatestCommitToLibraries(ref, null) ?: return@lazy ref
3535
resolvedSha
3636
}
3737

‎jupyter-lib/shared-compiler/src/main/kotlin/org/jetbrains/kotlinx/jupyter/libraries/ResolutionInfoProvider.kt

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,7 @@ class StandardResolutionInfoProvider(override var fallback: LibraryResolutionInf
4242
}
4343

4444
private fun tryGetAsRef(ref: String): LibraryResolutionInfo? {
45-
val response = getHttp("$GitHubApiPrefix/contents/$RemoteLibrariesDir?ref=$ref")
46-
return if (response.status.successful) LibraryResolutionInfo.getInfoByRef(ref) else null
45+
return if (KERNEL_LIBRARIES.checkRefExistence(ref)) LibraryResolutionInfo.getInfoByRef(ref) else null
4746
}
4847

4948
private fun tryGetAsDir(dirName: String): LibraryResolutionInfo? {

‎jupyter-lib/shared-compiler/src/main/kotlin/org/jetbrains/kotlinx/jupyter/libraries/constants.kt

Lines changed: 0 additions & 17 deletions
This file was deleted.

‎jupyter-lib/shared-compiler/src/main/kotlin/org/jetbrains/kotlinx/jupyter/libraries/httpUtil.kt

Lines changed: 0 additions & 33 deletions
This file was deleted.

‎jupyter-lib/shared-compiler/src/main/kotlin/org/jetbrains/kotlinx/jupyter/libraries/libraryParsingUtil.kt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,4 +40,4 @@ private val defaultParsers = listOf(
4040
LibraryResolutionInfoParser.make("url", listOf(Parameter.Required("url"))) { args ->
4141
LibraryResolutionInfo.ByURL(URL(args["url"] ?: error("Argument 'url' should be specified")))
4242
},
43-
).map { it.name to it }.toMap()
43+
).associateBy { it.name }

‎jupyter-lib/shared-compiler/src/main/kotlin/org/jetbrains/kotlinx/jupyter/libraries/libraryResolvers.kt

Lines changed: 10 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,11 @@
11
package org.jetbrains.kotlinx.jupyter.libraries
22

3-
import kotlinx.serialization.json.JsonPrimitive
43
import org.jetbrains.kotlinx.jupyter.api.libraries.LibraryDefinition
54
import org.jetbrains.kotlinx.jupyter.common.getHttp
6-
import org.jetbrains.kotlinx.jupyter.common.jsonObject
75
import org.jetbrains.kotlinx.jupyter.common.text
86
import org.jetbrains.kotlinx.jupyter.config.getLogger
97
import org.jetbrains.kotlinx.jupyter.exceptions.ReplLibraryLoadingException
10-
import java.nio.file.Path
8+
import java.io.File
119
import kotlin.reflect.KClass
1210
import kotlin.reflect.cast
1311

@@ -39,9 +37,9 @@ abstract class LibraryDescriptorResolver(private val parent: LibraryResolver? =
3937
class DefaultInfoLibraryResolver(
4038
private val parent: LibraryResolver,
4139
private val infoProvider: ResolutionInfoProvider,
42-
paths: List<Path> = emptyList(),
40+
paths: List<File> = emptyList(),
4341
) : LibraryResolver {
44-
private val resolutionInfos = paths.map { LibraryResolutionInfo.ByDir(it.toFile()) }
42+
private val resolutionInfos = paths.map { LibraryResolutionInfo.ByDir(it) }
4543

4644
override fun resolve(reference: LibraryReference, arguments: List<Variable>): LibraryDefinition? {
4745
val referenceInfo = reference.info
@@ -62,15 +60,15 @@ class DefaultInfoLibraryResolver(
6260

6361
class LocalLibraryResolver(
6462
parent: LibraryResolver?,
65-
mainLibrariesDir: Path?
63+
mainLibrariesDir: File?
6664
) : LibraryDescriptorResolver(parent) {
6765
private val logger = getLogger()
68-
private val pathsToCheck: List<Path>
66+
private val pathsToCheck: List<File>
6967

7068
init {
7169
val paths = mutableListOf(
72-
LocalSettingsPath.resolve(LocalCacheDir),
73-
LocalSettingsPath.resolve(LibrariesDir)
70+
KERNEL_LIBRARIES.userCacheDir,
71+
KERNEL_LIBRARIES.userLibrariesDir,
7472
)
7573
mainLibrariesDir?.let { paths.add(it) }
7674

@@ -103,22 +101,15 @@ class LocalLibraryResolver(
103101
file.writeText(text)
104102
}
105103

106-
private fun LibraryReference.getFile(dir: Path) = dir.resolve("$key.$LibraryDescriptorExt").toFile()
104+
private fun LibraryReference.getFile(dir: File) = dir.resolve(KERNEL_LIBRARIES.descriptorFileName(key))
107105
}
108106

109107
object FallbackLibraryResolver : LibraryDescriptorResolver() {
110108
private val standardResolvers = listOf(
111109
byDirResolver,
112110
resolver<LibraryResolutionInfo.ByGitRef> { name ->
113111
if (name == null) throw ReplLibraryLoadingException(message = "Reference library resolver needs name to be specified")
114-
115-
val url = "$GitHubApiPrefix/contents/$RemoteLibrariesDir/$name.$LibraryDescriptorExt?ref=$sha"
116-
getLogger().info("Requesting library descriptor at $url")
117-
val response = getHttp(url).jsonObject
118-
119-
val downloadURL = (response["download_url"] as JsonPrimitive).content
120-
val res = getHttp(downloadURL)
121-
res.text
112+
KERNEL_LIBRARIES.downloadLibraryDescriptor(sha, name)
122113
},
123114
resolver<LibraryResolutionInfo.ByFile> {
124115
file.readText()
@@ -153,7 +144,7 @@ private inline fun <reified T : LibraryResolutionInfo> resolver(noinline resolve
153144
private val byDirResolver = resolver<LibraryResolutionInfo.ByDir> { name ->
154145
if (name == null) throw ReplLibraryLoadingException(name, "Directory library resolver needs library name to be specified")
155146

156-
val jsonFile = librariesDir.resolve("$name.$LibraryDescriptorExt")
147+
val jsonFile = librariesDir.resolve(KERNEL_LIBRARIES.descriptorFileName(name))
157148
if (jsonFile.exists() && jsonFile.isFile) jsonFile.readText()
158149
else null
159150
}

‎jupyter-lib/shared-compiler/src/main/kotlin/org/jetbrains/kotlinx/jupyter/libraries/util.kt

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,20 @@ import org.jetbrains.kotlinx.jupyter.api.Code
44
import org.jetbrains.kotlinx.jupyter.api.Notebook
55
import org.jetbrains.kotlinx.jupyter.api.libraries.LibraryDefinition
66
import org.jetbrains.kotlinx.jupyter.api.libraries.LibraryDefinitionProducer
7+
import org.jetbrains.kotlinx.jupyter.common.LibraryDescriptorsManager
8+
import org.jetbrains.kotlinx.jupyter.config.errorForUser
9+
import org.jetbrains.kotlinx.jupyter.config.getLogger
710
import org.jetbrains.kotlinx.jupyter.util.replaceVariables
811
import java.io.File
9-
import java.nio.file.Path
10-
import java.nio.file.Paths
1112
import kotlin.script.experimental.api.ResultWithDiagnostics
1213
import kotlin.script.experimental.api.ScriptDiagnostic
1314

15+
val KERNEL_LIBRARIES = LibraryDescriptorsManager.getInstance(
16+
getLogger()
17+
) { logger, message, exception ->
18+
logger.errorForUser(message = message, throwable = exception)
19+
}
20+
1421
sealed class Parameter(val name: String, open val default: String?) {
1522
class Required(name: String) : Parameter(name, null)
1623
class Optional(name: String, override val default: String) : Parameter(name, default)
@@ -131,9 +138,9 @@ fun parseLibraryName(str: String): Pair<String, List<Variable>> {
131138
fun getStandardResolver(homeDir: String? = null, infoProvider: ResolutionInfoProvider): LibraryResolver {
132139
// Standard resolver doesn't cache results in memory
133140
var res: LibraryResolver = FallbackLibraryResolver
134-
val librariesDir: Path? = homeDir?.let { Paths.get(it, LibrariesDir) }
141+
val librariesDir: File? = homeDir?.let { KERNEL_LIBRARIES.homeLibrariesDir(File(it)) }
135142
res = LocalLibraryResolver(res, librariesDir)
136-
res = DefaultInfoLibraryResolver(res, infoProvider, listOf(LocalSettingsPath.resolve(LibrariesDir)))
143+
res = DefaultInfoLibraryResolver(res, infoProvider, listOf(KERNEL_LIBRARIES.userLibrariesDir))
137144
return res
138145
}
139146

‎src/main/kotlin/org/jetbrains/kotlinx/jupyter/commands.kt

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,7 @@ import org.jetbrains.kotlinx.jupyter.common.assertLooksLikeReplCommand
88
import org.jetbrains.kotlinx.jupyter.common.replCommandOrNull
99
import org.jetbrains.kotlinx.jupyter.compiler.util.SourceCodeImpl
1010
import org.jetbrains.kotlinx.jupyter.config.catchAll
11-
import org.jetbrains.kotlinx.jupyter.libraries.LibrariesDir
12-
import org.jetbrains.kotlinx.jupyter.libraries.LibraryDescriptorExt
11+
import org.jetbrains.kotlinx.jupyter.libraries.KERNEL_LIBRARIES
1312
import org.jetbrains.kotlinx.jupyter.libraries.parseLibraryDescriptor
1413
import org.jetbrains.kotlinx.jupyter.repl.CompletionResult
1514
import org.jetbrains.kotlinx.jupyter.repl.KotlinCompleter
@@ -67,8 +66,9 @@ fun runCommand(code: String, repl: ReplForJupyter): Response {
6766
if (it.argumentsUsage != null) s += "\n Usage: %${it.nameForUser} ${it.argumentsUsage}"
6867
s
6968
}
70-
val libraryFiles =
71-
repl.homeDir?.resolve(LibrariesDir)?.listFiles { file -> file.isFile && file.name.endsWith(".$LibraryDescriptorExt") } ?: emptyArray()
69+
val libraryFiles = repl.homeDir?.let { homeDir ->
70+
KERNEL_LIBRARIES.homeLibrariesDir(homeDir).listFiles(KERNEL_LIBRARIES::isLibraryDescriptor)
71+
} ?: emptyArray()
7272
val libraries = libraryFiles.toList().mapNotNull { file ->
7373
val libraryName = file.nameWithoutExtension
7474
log.info("Parsing descriptor for library '$libraryName'")

‎src/main/kotlin/org/jetbrains/kotlinx/jupyter/ikotlin.kt

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
package org.jetbrains.kotlinx.jupyter
22

33
import org.jetbrains.kotlinx.jupyter.libraries.EmptyResolutionInfoProvider
4-
import org.jetbrains.kotlinx.jupyter.libraries.LibrariesDir
4+
import org.jetbrains.kotlinx.jupyter.libraries.KERNEL_LIBRARIES
55
import org.jetbrains.kotlinx.jupyter.libraries.ResolutionInfoProvider
66
import java.io.File
77
import java.util.concurrent.atomic.AtomicLong
@@ -68,7 +68,7 @@ fun main(vararg args: String) {
6868
try {
6969
log.info("Kernel args: " + args.joinToString { it })
7070
val kernelArgs = parseCommandLine(*args)
71-
val libraryPath = (kernelArgs.homeDir ?: File("")).resolve(LibrariesDir)
71+
val libraryPath = KERNEL_LIBRARIES.homeLibrariesDir(kernelArgs.homeDir)
7272
val libraryInfoProvider = ResolutionInfoProvider.withDefaultDirectoryResolution(libraryPath)
7373
val kernelConfig = KernelConfig.fromArgs(kernelArgs, libraryInfoProvider)
7474
kernelServer(kernelConfig)

0 commit comments

Comments
 (0)