Skip to content

Commit 83dcee1

Browse files
jpenillajpenilla
committed
Improve userdev cache cleanup strategy
For now, new config properties are experimental, v1 cleanup is still implemented, and the old property is also used as the fallback value for the new expiry time property.
1 parent 2c814fb commit 83dcee1

File tree

8 files changed

+233
-138
lines changed

8 files changed

+233
-138
lines changed

‎paperweight-lib/src/main/kotlin/io/papermc/paperweight/util/utils.kt‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -483,11 +483,12 @@ fun ioDispatcher(name: String): ExecutorCoroutineDispatcher =
483483
Executors.newFixedThreadPool(
484484
Runtime.getRuntime().availableProcessors(),
485485
object : ThreadFactory {
486-
val logger = Logging.getLogger("$name-ioDispatcher-${ioDispatcherCount.getAndIncrement()}")
486+
val id = ioDispatcherCount.getAndIncrement()
487+
val logger = Logging.getLogger("$name-ioDispatcher-$id")
487488
val count = AtomicInteger(0)
488489

489490
override fun newThread(r: Runnable): Thread {
490-
val thr = Thread(r, "$name-ioDispatcher-${ioDispatcherCount.getAndIncrement()}-Thread-${count.getAndIncrement()}")
491+
val thr = Thread(r, "$name-ioDispatcher-$id-Thread-${count.getAndIncrement()}")
491492
thr.setUncaughtExceptionHandler { thread, ex ->
492493
logger.error("Uncaught exception in thread $thread", ex)
493494
}

‎paperweight-userdev/src/main/kotlin/io/papermc/paperweight/userdev/PaperweightUser.kt‎

Lines changed: 16 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,10 @@ import io.papermc.paperweight.userdev.internal.setup.SetupHandler
3333
import io.papermc.paperweight.userdev.internal.setup.UserdevSetup
3434
import io.papermc.paperweight.userdev.internal.setup.UserdevSetupTask
3535
import io.papermc.paperweight.userdev.internal.util.cleanSharedCaches
36-
import io.papermc.paperweight.userdev.internal.util.deleteUnusedAfter
36+
import io.papermc.paperweight.userdev.internal.util.delayCleanupBy
37+
import io.papermc.paperweight.userdev.internal.util.expireUnusedAfter
3738
import io.papermc.paperweight.userdev.internal.util.genSources
39+
import io.papermc.paperweight.userdev.internal.util.performCleanupAfter
3840
import io.papermc.paperweight.userdev.internal.util.sharedCaches
3941
import io.papermc.paperweight.util.*
4042
import io.papermc.paperweight.util.constants.*
@@ -350,17 +352,20 @@ abstract class PaperweightUser : Plugin<Project> {
350352
}
351353

352354
val serviceName = "paperweight-userdev:setupService:$bundleHash"
353-
val ret = target.gradle.sharedServices
354-
.registerIfAbsent(serviceName, UserdevSetup::class) {
355-
parameters {
356-
cache.set(cacheDir)
357-
bundleZip.set(devBundleZip)
358-
bundleZipHash.set(bundleHash)
359-
downloadService.set(target.download)
360-
genSources.set(target.genSources)
361-
deleteUnusedAfter.set(deleteUnusedAfter(target))
362-
}
355+
val ret = target.gradle.sharedServices.registerIfAbsent(serviceName, UserdevSetup::class) {
356+
parameters {
357+
cache.set(cacheDir)
358+
downloadService.set(target.download)
359+
genSources.set(target.genSources)
360+
361+
bundleZip.set(devBundleZip)
362+
bundleZipHash.set(bundleHash)
363+
364+
expireUnusedAfter.set(expireUnusedAfter(target))
365+
performCleanupAfter.set(performCleanupAfter(target))
366+
delayCleanupBy.set(delayCleanupBy(target))
363367
}
368+
}
364369
buildEventsListenerRegistry.onTaskCompletion(ret)
365370
return ret
366371
}

‎paperweight-userdev/src/main/kotlin/io/papermc/paperweight/userdev/internal/action/CacheCleaner.kt‎

Lines changed: 0 additions & 89 deletions
This file was deleted.

‎paperweight-userdev/src/main/kotlin/io/papermc/paperweight/userdev/internal/action/CacheManager.kt‎

Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
/*
2+
* paperweight is a Gradle plugin for the PaperMC project.
3+
*
4+
* Copyright (c) 2023 Kyle Wood (DenWav)
5+
* Contributors
6+
*
7+
* This library is free software; you can redistribute it and/or
8+
* modify it under the terms of the GNU Lesser General Public
9+
* License as published by the Free Software Foundation;
10+
* version 2.1 only, no later versions.
11+
*
12+
* This library is distributed in the hope that it will be useful,
13+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
14+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15+
* Lesser General Public License for more details.
16+
*
17+
* You should have received a copy of the GNU Lesser General Public
18+
* License along with this library; if not, write to the Free Software
19+
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301
20+
* USA
21+
*/
22+
23+
package io.papermc.paperweight.userdev.internal.action
24+
25+
import io.papermc.paperweight.userdev.internal.util.formatNs
26+
import io.papermc.paperweight.util.*
27+
import java.nio.file.Files
28+
import java.nio.file.Path
29+
import java.time.Instant
30+
import kotlin.io.path.*
31+
import org.gradle.api.logging.LogLevel
32+
import org.gradle.api.logging.Logging
33+
34+
class CacheManager(private val root: Path) {
35+
companion object {
36+
private val logger = Logging.getLogger(CacheManager::class.java)
37+
}
38+
39+
data class MaintenanceInfo(
40+
val lastCleanup: Long = System.currentTimeMillis(),
41+
val scheduledCleanup: Long? = null,
42+
) {
43+
fun writeTo(file: Path) {
44+
file.createParentDirectories().writeText(gson.toJson(this))
45+
}
46+
47+
companion object {
48+
fun readFrom(file: Path): MaintenanceInfo = gson.fromJson(file)
49+
}
50+
}
51+
52+
fun performMaintenance(
53+
expireUnusedAfter: Long,
54+
performCleanupAfter: Long,
55+
delayCleanupBy: Long,
56+
bundleZipHash: String,
57+
) {
58+
if (!root.isDirectory()) {
59+
return
60+
}
61+
62+
val maintenanceLock = root.resolve("maintenance.lock")
63+
val maintenanceFile = root.resolve("maintenance.json")
64+
65+
val start = System.nanoTime()
66+
67+
withLock(maintenanceLock) {
68+
logger.info("paperweight-userdev: Acquired cache maintenance lock in ${formatNs(System.nanoTime() - start)}")
69+
70+
// update last used for final outputs in case the task was up-to-date
71+
root.listDirectoryEntries().forEach { entry ->
72+
val metadataFile = entry.resolve(WorkGraph.METADATA_FILE)
73+
if (!entry.isDirectory() || !metadataFile.isRegularFile() || entry.resolve("lock").exists()) {
74+
return@forEach
75+
}
76+
if (entry.name.endsWith("_$bundleZipHash")) {
77+
gson.fromJson<WorkGraph.Metadata>(metadataFile).updateLastUsed().writeTo(metadataFile)
78+
}
79+
}
80+
81+
if (maintenanceFile.isRegularFile()) {
82+
val info = MaintenanceInfo.readFrom(maintenanceFile)
83+
if (System.currentTimeMillis() - info.lastCleanup < performCleanupAfter) {
84+
return@withLock
85+
}
86+
if (info.scheduledCleanup == null) {
87+
val cleanup = System.currentTimeMillis() + delayCleanupBy
88+
logger.info("paperweight-userdev: Scheduled cache cleanup for after ${Instant.ofEpochMilli(cleanup)}")
89+
info.copy(scheduledCleanup = cleanup).writeTo(maintenanceFile)
90+
} else if (System.currentTimeMillis() >= info.scheduledCleanup) {
91+
if (cleanup(expireUnusedAfter)) {
92+
info.copy(lastCleanup = System.currentTimeMillis(), scheduledCleanup = null).writeTo(maintenanceFile)
93+
}
94+
// else: cleanup was skipped due to locked cache entry, try again later
95+
}
96+
} else {
97+
MaintenanceInfo().writeTo(maintenanceFile)
98+
}
99+
}
100+
101+
logger.info("paperweight-userdev: Finished cache maintenance in ${formatNs(System.nanoTime() - start)}")
102+
}
103+
104+
private fun cleanup(deleteUnusedAfter: Long): Boolean {
105+
val tryDelete = mutableListOf<Path>()
106+
val keep = mutableListOf<Path>()
107+
root.listDirectoryEntries().forEach {
108+
val metadataFile = it.resolve(WorkGraph.METADATA_FILE)
109+
if (!metadataFile.isRegularFile()) {
110+
return@forEach
111+
}
112+
if (it.resolve("lock").exists()) {
113+
logger.info("paperweight-userdev: Aborted cache cleanup due to locked cache entry (${it.name})")
114+
return false
115+
}
116+
val since = System.currentTimeMillis() - metadataFile.getLastModifiedTime().toMillis()
117+
if (since > deleteUnusedAfter) {
118+
tryDelete.add(it)
119+
} else {
120+
keep.add(it)
121+
}
122+
}
123+
124+
var deleted = 0
125+
var deletedSize = 0L
126+
if (tryDelete.isNotEmpty()) {
127+
keep.forEach { k ->
128+
val metadataFile = k.resolve(WorkGraph.METADATA_FILE)
129+
gson.fromJson<WorkGraph.Metadata>(metadataFile).skippedWhenUpToDate?.let {
130+
tryDelete.removeIf { o -> o.name in it }
131+
}
132+
}
133+
134+
tryDelete.forEach {
135+
deleted++
136+
it.deleteRecursive { toDelete ->
137+
if (toDelete.isRegularFile()) {
138+
deletedSize += Files.size(toDelete)
139+
}
140+
}
141+
}
142+
}
143+
144+
val level = if (deleted > 0) LogLevel.LIFECYCLE else LogLevel.INFO
145+
logger.log(
146+
level,
147+
"paperweight-userdev: Deleted $deleted expired cache entries totaling ${deletedSize / 1024}KB"
148+
)
149+
return true
150+
}
151+
}

‎paperweight-userdev/src/main/kotlin/io/papermc/paperweight/userdev/internal/action/WorkGraph.kt‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ class WorkGraph(
4040
) {
4141
companion object {
4242
private val logger = Logging.getLogger(WorkGraph::class.java)
43+
const val METADATA_FILE = "metadata.json"
4344
}
4445

4546
class Node(
@@ -151,7 +152,7 @@ class WorkGraph(
151152

152153
val lockFile = work.resolve("${node.registration.name}_$inputHash/lock")
153154

154-
val metadataFile = work.resolve("${node.registration.name}_$inputHash/metadata.json")
155+
val metadataFile = work.resolve("${node.registration.name}_$inputHash/$METADATA_FILE")
155156
val upToDate = withLock(lockFile) {
156157
if (metadataFile.exists()) {
157158
val metadata = metadataFile.bufferedReader().use { gson.fromJson(it, Metadata::class.java) }

‎paperweight-userdev/src/main/kotlin/io/papermc/paperweight/userdev/internal/setup/SetupHandler.kt‎

Lines changed: 20 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -127,26 +127,26 @@ interface SetupHandler {
127127

128128
companion object {
129129
@Suppress("unchecked_cast")
130-
fun create(
131-
parameters: UserdevSetup.Parameters,
132-
bundleInfo: BundleInfo<Any>
133-
): SetupHandler = when (bundleInfo.config) {
134-
is GenerateDevBundle.DevBundleConfig -> SetupHandlerImpl(
135-
parameters,
136-
bundleInfo as BundleInfo<GenerateDevBundle.DevBundleConfig>,
137-
)
138-
139-
is DevBundleV5.Config -> SetupHandlerImplV5(
140-
parameters,
141-
bundleInfo as BundleInfo<DevBundleV5.Config>
142-
)
143-
144-
is DevBundleV2.Config -> SetupHandlerImplV2(
145-
parameters,
146-
bundleInfo as BundleInfo<DevBundleV2.Config>
147-
)
148-
149-
else -> throw PaperweightException("Unknown dev bundle config type: ${bundleInfo.config::class.java.typeName}")
130+
fun create(parameters: UserdevSetup.Parameters): SetupHandler {
131+
val bundleInfo = readBundleInfo(parameters.bundleZip.path)
132+
return when (bundleInfo.config) {
133+
is GenerateDevBundle.DevBundleConfig -> SetupHandlerImpl(
134+
parameters,
135+
bundleInfo as BundleInfo<GenerateDevBundle.DevBundleConfig>,
136+
)
137+
138+
is DevBundleV5.Config -> SetupHandlerImplV5(
139+
parameters,
140+
bundleInfo as BundleInfo<DevBundleV5.Config>
141+
)
142+
143+
is DevBundleV2.Config -> SetupHandlerImplV2(
144+
parameters,
145+
bundleInfo as BundleInfo<DevBundleV2.Config>
146+
)
147+
148+
else -> throw PaperweightException("Unknown dev bundle config type: ${bundleInfo.config::class.java.typeName}")
149+
}
150150
}
151151
}
152152
}

0 commit comments

Comments
 (0)