Skip to content

Commit 6586167

Browse files
adam-enkoadam-enko
committed
generate CDS for Dokka workers
1 parent 4b53ec1 commit 6586167

File tree

2 files changed

+194
-0
lines changed

2 files changed

+194
-0
lines changed

‎dokka-runners/dokka-gradle-plugin/src/main/kotlin/internal/CdsSource.kt‎

Lines changed: 178 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,178 @@
1+
/*
2+
* Copyright 2014-2025 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
3+
*/
4+
5+
package org.jetbrains.dokka.gradle.internal
6+
7+
import org.gradle.api.file.ConfigurableFileCollection
8+
import org.gradle.api.provider.ValueSource
9+
import org.gradle.api.provider.ValueSourceParameters
10+
import org.gradle.process.ExecOperations
11+
import org.jetbrains.kotlin.konan.file.use
12+
import java.io.File
13+
import java.io.OutputStream
14+
import java.io.RandomAccessFile
15+
import java.math.BigInteger
16+
import java.nio.channels.FileChannel
17+
import java.nio.channels.FileLock
18+
import java.nio.channels.OverlappingFileLockException
19+
import java.nio.file.Files
20+
import java.security.DigestOutputStream
21+
import java.security.MessageDigest
22+
import java.util.concurrent.locks.Lock
23+
import java.util.concurrent.locks.ReentrantLock
24+
import java.util.jar.JarFile
25+
import javax.inject.Inject
26+
import kotlin.concurrent.withLock
27+
28+
29+
internal abstract class CdsSource
30+
@Inject
31+
internal constructor(
32+
private val execOps: ExecOperations
33+
) : ValueSource<File, CdsSource.Parameters> {
34+
35+
interface Parameters : ValueSourceParameters {
36+
val classpath: ConfigurableFileCollection
37+
}
38+
39+
private val classpathChecksum: String by lazy {
40+
checksum(parameters.classpath)
41+
}
42+
43+
private val cacheDir: File by lazy {
44+
val osName = System.getProperty("os.name").lowercase()
45+
val homeDir = System.getProperty("user.home")
46+
val appDataDir = System.getenv("APP_DATA") ?: homeDir
47+
48+
val userCacheDir = when {
49+
"win" in osName -> "$appDataDir/Caches/"
50+
"mac" in osName -> "$homeDir/Library/Caches/"
51+
"nix" in osName -> "$homeDir/.cache/"
52+
else -> "$homeDir/.cache/"
53+
}
54+
55+
File(userCacheDir).resolve("dokka").apply {
56+
mkdirs()
57+
}
58+
}
59+
60+
private val cdsFile: File by lazy {
61+
cacheDir.resolve("$classpathChecksum.jsa")
62+
}
63+
private val lockFile: File by lazy {
64+
cacheDir.resolve("$classpathChecksum.lock")
65+
}
66+
67+
override fun obtain(): File {
68+
lock.withLock {
69+
RandomAccessFile(lockFile, "rw").use {
70+
it.channel.lockWithRetries().use {
71+
if (!cdsFile.exists()) {
72+
generateStaticCds()
73+
}
74+
println("Using CDS ${cdsFile.absoluteFile.invariantSeparatorsPath}")
75+
return cdsFile
76+
}
77+
}
78+
}
79+
}
80+
81+
private fun generateStaticCds() {
82+
83+
val classListFile = Files.createTempFile("asd", "classlist").toFile()
84+
parameters.classpath.files.flatMap { file ->
85+
getClassNamesFromJarFile(file)
86+
}
87+
.toSet()
88+
.joinToString("\n")
89+
.let {
90+
classListFile.writeText(it)
91+
}
92+
93+
execOps.javaexec {
94+
jvmArgs(
95+
"-Xshare:dump",
96+
"-XX:SharedArchiveFile=${cdsFile.absoluteFile.invariantSeparatorsPath}",
97+
"-XX:SharedClassListFile=${classListFile.absoluteFile.invariantSeparatorsPath}"
98+
)
99+
classpath(parameters.classpath)
100+
}
101+
}
102+
103+
companion object {
104+
private val lock: Lock = ReentrantLock()
105+
}
106+
}
107+
108+
109+
private fun checksum(
110+
files: ConfigurableFileCollection
111+
): String {
112+
val md = MessageDigest.getInstance("md5")
113+
DigestOutputStream(nullOutputStream(), md).use { os ->
114+
os.write(files.asPath.encodeToByteArray())
115+
}
116+
return BigInteger(1, md.digest()).toString(16)
117+
.padStart(md.digestLength * 2, '0')
118+
}
119+
120+
private fun checksum(
121+
files: Collection<File>
122+
): String {
123+
val md = MessageDigest.getInstance("md5")
124+
DigestOutputStream(nullOutputStream(), md).use { os ->
125+
files.forEach { file ->
126+
file.inputStream().use { it.copyTo(os) }
127+
}
128+
}
129+
return BigInteger(1, md.digest()).toString(16)
130+
.padStart(md.digestLength * 2, '0')
131+
}
132+
133+
private fun nullOutputStream(): OutputStream =
134+
object : OutputStream() {
135+
override fun write(b: Int) {}
136+
}
137+
138+
139+
private fun getClassNamesFromJarFile(source: File): Set<String> {
140+
JarFile(source).use { jarFile ->
141+
return jarFile.entries().asSequence()
142+
.filter { it.name.endsWith(".class") }
143+
.map { entry ->
144+
entry.name
145+
.replace("/", ".")
146+
.removeSuffix(".class")
147+
}
148+
.toSet()
149+
}
150+
}
151+
152+
private fun FileChannel.lockWithRetries(): FileLock {
153+
var retries = 0
154+
while (true) {
155+
try {
156+
return lock()
157+
}
158+
/*
159+
Catching the OverlappingFileLockException which is caused by the same jvm (process) already having locked the file.
160+
Since we do use a static re-entrant lock as a monitor to the cache, this can only happen
161+
when this code is running in the same JVM but with in complete isolation
162+
(e.g. Gradle classpath isolation, or composite builds).
163+
164+
If we detect this case, we retry the locking after a short period, constantly logging that we're blocked
165+
by some other thread using the cache.
166+
167+
The risk of deadlocking here is low, since we can only get into this code path, *if*
168+
the code is very isolated and somebody locked the file.
169+
*/
170+
catch (t: OverlappingFileLockException) {
171+
Thread.sleep(25)
172+
retries++
173+
// if (retries % 10 == 0) {
174+
//// logInfo("Waiting to acquire lock: $file")
175+
// }
176+
}
177+
}
178+
}

‎dokka-runners/dokka-gradle-plugin/src/main/kotlin/tasks/DokkaGenerateTask.kt‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,15 +10,18 @@ import org.gradle.api.file.DirectoryProperty
1010
import org.gradle.api.file.RegularFileProperty
1111
import org.gradle.api.model.ObjectFactory
1212
import org.gradle.api.provider.Property
13+
import org.gradle.api.provider.ProviderFactory
1314
import org.gradle.api.tasks.*
1415
import org.gradle.kotlin.dsl.newInstance
16+
import org.gradle.kotlin.dsl.of
1517
import org.gradle.kotlin.dsl.submit
1618
import org.gradle.workers.WorkerExecutor
1719
import org.jetbrains.dokka.DokkaConfiguration
1820
import org.jetbrains.dokka.DokkaConfigurationImpl
1921
import org.jetbrains.dokka.gradle.DokkaBasePlugin.Companion.jsonMapper
2022
import org.jetbrains.dokka.gradle.engine.parameters.DokkaGeneratorParametersSpec
2123
import org.jetbrains.dokka.gradle.engine.parameters.builders.DokkaParametersBuilder
24+
import org.jetbrains.dokka.gradle.internal.CdsSource
2225
import org.jetbrains.dokka.gradle.internal.DokkaPluginParametersContainer
2326
import org.jetbrains.dokka.gradle.internal.InternalDokkaGradlePluginApi
2427
import org.jetbrains.dokka.gradle.workers.ClassLoaderIsolation
@@ -50,6 +53,10 @@ constructor(
5053
pluginsConfiguration: DokkaPluginParametersContainer,
5154
) : DokkaBaseTask() {
5255

56+
@InternalDokkaGradlePluginApi
57+
@get:Inject
58+
protected open val providers: ProviderFactory get() = error("injected")
59+
5360
private val dokkaParametersBuilder = DokkaParametersBuilder(archives)
5461

5562
/**
@@ -160,6 +167,15 @@ constructor(
160167
isolation.minHeapSize.orNull?.let(this::setMinHeapSize)
161168
isolation.jvmArgs.orNull?.filter { it.isNotBlank() }?.let(this::setJvmArgs)
162169
isolation.systemProperties.orNull?.let(this::systemProperties)
170+
171+
val cds = providers.of(CdsSource::class) {
172+
parameters {
173+
classpath.from(runtimeClasspath)
174+
}
175+
}
176+
jvmArgs(
177+
"-XX:SharedArchiveFile=${cds.get().absoluteFile.invariantSeparatorsPath}"
178+
)
163179
}
164180
}
165181
}

0 commit comments

Comments
 (0)