Skip to content

Commit 2c814fb

Browse files
jpenillajpenilla
committed
Persist 'expired' userdev cache entries when a non-expired entry uses it
Useful because we don't update the last used time for intermediate outputs when the final output is up to date.
1 parent 21b1e82 commit 2c814fb

File tree

3 files changed

+71
-32
lines changed

3 files changed

+71
-32
lines changed

‎paperweight-userdev/src/main/kotlin/io/papermc/paperweight/userdev/internal/action/CacheCleaner.kt‎

Lines changed: 25 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222

2323
package io.papermc.paperweight.userdev.internal.action
2424

25+
import io.papermc.paperweight.userdev.internal.util.formatNs
2526
import io.papermc.paperweight.util.*
2627
import java.nio.file.Files
2728
import java.nio.file.Path
@@ -40,20 +41,38 @@ class CacheCleaner(private val work: Path) {
4041
}
4142

4243
val start = System.nanoTime()
43-
var deleted = 0
44-
var deletedSize = 0L
4544

45+
val delete = mutableListOf<Path>()
46+
val keep = mutableListOf<Path>()
4647
work.listDirectoryEntries().forEach {
47-
val lockFile = it.resolve("lock")
48-
if (lockFile.exists()) {
49-
return@forEach
48+
if (it.resolve("lock").exists()) {
49+
val took = System.nanoTime() - start
50+
logger.info("paperweight-userdev: Aborted cache cleanup in ${formatNs(took)} due to locked cache entry (${it.name})")
51+
return
5052
}
5153
val metadataFile = it.resolve("metadata.json")
5254
if (!metadataFile.isRegularFile()) {
5355
return@forEach
5456
}
5557
val since = System.currentTimeMillis() - metadataFile.getLastModifiedTime().toMillis()
5658
if (since > deleteUnusedAfter) {
59+
delete.add(it)
60+
} else {
61+
keep.add(it)
62+
}
63+
}
64+
65+
var deleted = 0
66+
var deletedSize = 0L
67+
if (delete.isNotEmpty()) {
68+
keep.forEach { k ->
69+
val metadataFile = k.resolve("metadata.json")
70+
gson.fromJson<WorkGraph.Metadata>(metadataFile).skippedWhenUpToDate?.let {
71+
delete.removeIf { o -> o.name in it }
72+
}
73+
}
74+
75+
delete.forEach {
5776
deleted++
5877
it.deleteRecursive { toDelete ->
5978
if (toDelete.isRegularFile()) {
@@ -65,6 +84,6 @@ class CacheCleaner(private val work: Path) {
6584

6685
val took = System.nanoTime() - start
6786
val level = if (deleted > 0) LogLevel.LIFECYCLE else LogLevel.INFO
68-
logger.log(level, "paperweight-userdev: Deleted $deleted expired cache entries totaling ${deletedSize / 1024}KB in ${took / 1_000_000}ms")
87+
logger.log(level, "paperweight-userdev: Deleted $deleted expired cache entries totaling ${deletedSize / 1024}KB in ${formatNs(took)}")
6988
}
7089
}

‎paperweight-userdev/src/main/kotlin/io/papermc/paperweight/userdev/internal/action/WorkGraph.kt‎

Lines changed: 26 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ class WorkGraph(
4545
class Node(
4646
val registration: WorkDispatcherImpl.Registration,
4747
val dependencies: List<Node>,
48+
var inputHash: String? = null,
4849
)
4950

5051
private val roots: List<Node> = buildGraph(requested.toList())
@@ -101,6 +102,7 @@ class WorkGraph(
101102

102103
data class Metadata(
103104
val outputHashes: List<String>,
105+
val skippedWhenUpToDate: Set<String>?,
104106
val lastUsed: Long = System.currentTimeMillis(),
105107
) {
106108
fun updateLastUsed() = copy(lastUsed = System.currentTimeMillis())
@@ -143,18 +145,19 @@ class WorkGraph(
143145
.hash(HashingAlgorithm.SHA256)
144146
.asHexString()
145147
}
148+
node.inputHash = inputHash
146149

147150
realizeOutputPaths(node, work, inputHash)
148151

149152
val lockFile = work.resolve("${node.registration.name}_$inputHash/lock")
150153

151-
val hashFile = work.resolve("${node.registration.name}_$inputHash/metadata.json")
154+
val metadataFile = work.resolve("${node.registration.name}_$inputHash/metadata.json")
152155
val upToDate = withLock(lockFile) {
153-
if (hashFile.exists()) {
154-
val metadata = hashFile.bufferedReader().use { gson.fromJson(it, Metadata::class.java) }
156+
if (metadataFile.exists()) {
157+
val metadata = metadataFile.bufferedReader().use { gson.fromJson(it, Metadata::class.java) }
155158
if (node.registration.outputs.hash(hashCache) == metadata.outputHashes) {
156159
logger.lifecycle("Skipping ${node.registration.name} (up-to-date)")
157-
metadata.updateLastUsed().writeTo(hashFile)
160+
metadata.updateLastUsed().writeTo(metadataFile)
158161
val took = System.nanoTime() - start
159162
logger.info("Up-to-date check for ${node.registration.name} took ${formatNs(took)} (up-to-date)")
160163
return@withLock true
@@ -175,8 +178,9 @@ class WorkGraph(
175178
throw PaperweightException("Exception executing ${node.registration.name}", e)
176179
}
177180

178-
val metadata = Metadata(node.registration.outputs.hash(hashCache))
179-
metadata.writeTo(hashFile)
181+
val deps = if (root && terminalInputHash != null) collectDependencies(node) else null
182+
val metadata = Metadata(node.registration.outputs.hash(hashCache), deps?.takeIf { it.isNotEmpty() })
183+
metadata.writeTo(metadataFile)
180184

181185
val tookExec = System.nanoTime() - startExec
182186
logger.lifecycle("Finished ${node.registration.name} in ${formatNs(tookExec)}")
@@ -193,6 +197,22 @@ class WorkGraph(
193197
}
194198
}
195199

200+
private fun collectDependencies(node: Node): Set<String> {
201+
val deps = mutableSetOf<String>()
202+
val nodes = mutableListOf<Node>()
203+
nodes.add(node)
204+
while (nodes.isNotEmpty()) {
205+
val n = nodes.removeAt(0)
206+
for (dep in n.dependencies) {
207+
nodes.add(dep)
208+
if (dep != node) {
209+
deps += "${dep.registration.name}_${dep.inputHash}"
210+
}
211+
}
212+
}
213+
return deps
214+
}
215+
196216
private fun List<Value<*>>.hash(cache: MutableMap<Value<*>, String>): List<String> {
197217
return map {
198218
cache.computeIfAbsent(it) { v ->

‎paperweight-userdev/src/main/kotlin/io/papermc/paperweight/userdev/internal/setup/action/VanillaServerDownloads.kt‎

Lines changed: 20 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -63,28 +63,28 @@ class VanillaServerDownloads(
6363
)
6464
val versionManifest: MinecraftVersionManifest = gson.fromJson(versionManifestPath)
6565

66-
val dispatcher = ioDispatcher("VanillaServerDownloads")
67-
runBlocking {
68-
launch(dispatcher) {
69-
val serverTmp = tmp.resolve("server.jar")
70-
downloadService.downloadFile(
71-
versionManifest.serverDownload().url,
72-
serverTmp,
73-
expectedHash = versionManifest.serverDownload().hash()
74-
)
75-
serverTmp.copyTo(serverJar.get(), overwrite = true)
76-
}
77-
launch(dispatcher) {
78-
val mappingsTmp = tmp.resolve("mappings.txt")
79-
downloadService.downloadFile(
80-
versionManifest.serverMappingsDownload().url,
81-
mappingsTmp,
82-
expectedHash = versionManifest.serverMappingsDownload().hash()
83-
)
84-
mappingsTmp.copyTo(serverMappings.get(), overwrite = true)
66+
ioDispatcher("VanillaServerDownloads").use { dispatcher ->
67+
runBlocking {
68+
launch(dispatcher) {
69+
val serverTmp = tmp.resolve("server.jar")
70+
downloadService.downloadFile(
71+
versionManifest.serverDownload().url,
72+
serverTmp,
73+
expectedHash = versionManifest.serverDownload().hash()
74+
)
75+
serverTmp.copyTo(serverJar.get(), overwrite = true)
76+
}
77+
launch(dispatcher) {
78+
val mappingsTmp = tmp.resolve("mappings.txt")
79+
downloadService.downloadFile(
80+
versionManifest.serverMappingsDownload().url,
81+
mappingsTmp,
82+
expectedHash = versionManifest.serverMappingsDownload().hash()
83+
)
84+
mappingsTmp.copyTo(serverMappings.get(), overwrite = true)
85+
}
8586
}
8687
}
87-
dispatcher.close()
8888

8989
tmp.deleteRecursive()
9090
}

0 commit comments

Comments
 (0)