Skip to content

Commit 0d19abf

Browse files
author
Oleksandr Dzhychko
committed
perf(model-server): introduce and use suspendable transactions
1 parent 5fc4f9c commit 0d19abf

File tree

6 files changed

+61
-41
lines changed

6 files changed

+61
-41
lines changed

‎model-server/src/main/kotlin/org/modelix/model/server/handlers/HistoryHandler.kt‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,8 @@ class HistoryHandler(val client: IModelClient, private val repositoriesManager:
6868
val params = call.request.queryParameters
6969
val limit = toInt(params["limit"], 500)
7070
val skip = toInt(params["skip"], 0)
71+
val latestVersion = repositoriesManager.getVersion(branch)
72+
checkNotNull(latestVersion) { "Branch not found: $branch" }
7173
call.respondHtmlTemplate(PageWithMenuBar("repos/", "../../..")) {
7274
headContent {
7375
style {
@@ -80,7 +82,7 @@ class HistoryHandler(val client: IModelClient, private val repositoriesManager:
8082
repositoryPageStyle()
8183
}
8284
bodyContent {
83-
buildRepositoryPage(branch, params["head"], skip, limit)
85+
buildRepositoryPage(branch, latestVersion, params["head"], skip, limit)
8486
}
8587
}
8688
}
@@ -105,7 +107,7 @@ class HistoryHandler(val client: IModelClient, private val repositoriesManager:
105107
}
106108
}
107109

108-
fun revert(repositoryAndBranch: BranchReference, from: String?, to: String?, author: String?) {
110+
suspend fun revert(repositoryAndBranch: BranchReference, from: String?, to: String?, author: String?) {
109111
val version = repositoriesManager.getVersion(repositoryAndBranch) ?: throw RuntimeException("Branch doesn't exist: $repositoryAndBranch")
110112
val branch = OTBranch(PBranch(version.tree, client.idGenerator), client.idGenerator, client.storeCache!!)
111113
branch.runWriteT { t ->
@@ -160,8 +162,13 @@ class HistoryHandler(val client: IModelClient, private val repositoriesManager:
160162
}
161163
}
162164

163-
private fun FlowContent.buildRepositoryPage(repositoryAndBranch: BranchReference, headHash: String?, skip: Int, limit: Int) {
164-
val latestVersion = repositoriesManager.getVersion(repositoryAndBranch) ?: throw RuntimeException("Branch not found: $repositoryAndBranch")
165+
private fun FlowContent.buildRepositoryPage(
166+
repositoryAndBranch: BranchReference,
167+
latestVersion: CLVersion,
168+
headHash: String?,
169+
skip: Int,
170+
limit: Int,
171+
) {
165172
val headVersion = if (headHash == null || headHash.length == 0) latestVersion else CLVersion(headHash, client.storeCache!!)
166173
var rowIndex = 0
167174
h1 {

‎model-server/src/main/kotlin/org/modelix/model/server/handlers/KeyValueLikeModelServer.kt‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import io.ktor.server.resources.put
2727
import io.ktor.server.response.respondText
2828
import io.ktor.server.routing.routing
2929
import io.ktor.util.pipeline.PipelineContext
30+
import kotlinx.coroutines.runBlocking
3031
import kotlinx.html.br
3132
import kotlinx.html.div
3233
import kotlinx.html.h1
@@ -48,12 +49,12 @@ import org.modelix.model.lazy.RepositoryId
4849
import org.modelix.model.persistent.HashUtil
4950
import org.modelix.model.server.store.IStoreClient
5051
import org.modelix.model.server.store.pollEntry
52+
import org.modelix.model.server.store.runTransactionSuspendable
5153
import org.modelix.model.server.templates.PageWithMenuBar
5254
import org.slf4j.LoggerFactory
5355
import java.io.IOException
5456
import java.util.*
5557
import java.util.regex.Pattern
56-
import kotlin.collections.LinkedHashMap
5758

5859
val PERMISSION_MODEL_SERVER = "model-server".asResource()
5960
val MODEL_SERVER_ENTRY = KeycloakResourceType("model-server-entry", KeycloakScope.READ_WRITE_DELETE)
@@ -85,7 +86,7 @@ class KeyValueLikeModelServer(val repositoriesManager: RepositoriesManager) {
8586
// request to initialize it lazily, would make the code less robust.
8687
// Each change in the logic of RepositoriesManager#maybeInitAndGetSeverId would need
8788
// the special conditions in the affected requests to be updated.
88-
repositoriesManager.maybeInitAndGetSeverId()
89+
runBlocking { repositoriesManager.maybeInitAndGetSeverId() }
8990
application.apply {
9091
modelServerModule()
9192
}
@@ -283,7 +284,7 @@ class KeyValueLikeModelServer(val repositoriesManager: RepositoriesManager) {
283284
return result
284285
}
285286

286-
protected fun CallContext.putEntries(newEntries: Map<String, String?>) {
287+
protected suspend fun CallContext.putEntries(newEntries: Map<String, String?>) {
287288
val referencedKeys: MutableSet<String> = HashSet()
288289
for ((key, value) in newEntries) {
289290
checkKeyPermission(key, EPermissionType.WRITE)
@@ -336,14 +337,14 @@ class KeyValueLikeModelServer(val repositoriesManager: RepositoriesManager) {
336337

337338
HashUtil.checkObjectHashes(hashedObjects)
338339

339-
repositoriesManager.client.store.runTransaction {
340+
repositoriesManager.client.store.runTransactionSuspendable {
340341
storeClient.putAll(hashedObjects)
341342
storeClient.putAll(userDefinedEntries)
342343
for ((branch, value) in branchChanges) {
343344
if (value == null) {
344-
repositoriesManager.removeBranches(branch.repositoryId, setOf(branch.branchName))
345+
runBlocking { repositoriesManager.removeBranches(branch.repositoryId, setOf(branch.branchName)) }
345346
} else {
346-
repositoriesManager.mergeChanges(branch, value)
347+
runBlocking { repositoriesManager.mergeChanges(branch, value) }
347348
}
348349
}
349350
}

‎model-server/src/main/kotlin/org/modelix/model/server/handlers/RepositoriesManager.kt‎

Lines changed: 35 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -42,12 +42,27 @@ import org.modelix.model.persistent.CPVersion
4242
import org.modelix.model.server.store.IStoreClient
4343
import org.modelix.model.server.store.LocalModelClient
4444
import org.modelix.model.server.store.pollEntry
45+
import org.modelix.model.server.store.runTransactionSuspendable
4546
import java.lang.ref.SoftReference
4647
import java.util.UUID
4748

4849
class RepositoriesManager(val client: LocalModelClient) {
4950

5051
init {
52+
fun migrateLegacyRepositoriesList() {
53+
val legacyRepositories = listLegacyRepositories().groupBy { it.repositoryId }
54+
if (legacyRepositories.isNotEmpty()) {
55+
// To not use `runTransactionSuspendable` like everywhere else,
56+
// because this is blocking initialization code anyways.
57+
store.runTransaction {
58+
ensureRepositoriesAreInList(legacyRepositories.keys)
59+
for ((legacyRepository, legacyBranches) in legacyRepositories) {
60+
ensureBranchesAreInList(legacyRepository, legacyBranches.map { it.branchName }.toSet())
61+
}
62+
}
63+
}
64+
}
65+
5166
migrateLegacyRepositoriesList()
5267
}
5368

@@ -75,8 +90,8 @@ class RepositoriesManager(val client: LocalModelClient) {
7590
* If the server ID was created previously but is only stored under a legacy database key,
7691
* it also gets stored under the current and all legacy database keys.
7792
*/
78-
fun maybeInitAndGetSeverId(): String {
79-
return store.runTransaction {
93+
suspend fun maybeInitAndGetSeverId(): String {
94+
return store.runTransactionSuspendable {
8095
var serverId = store[SERVER_ID_KEY]
8196
if (serverId == null) {
8297
serverId = store[LEGACY_SERVER_ID_KEY2]
@@ -102,9 +117,9 @@ class RepositoriesManager(val client: LocalModelClient) {
102117

103118
private fun repositoryExists(repositoryId: RepositoryId) = getRepositories().contains(repositoryId)
104119

105-
fun createRepository(repositoryId: RepositoryId, userName: String?, useRoleIds: Boolean = true): CLVersion {
120+
suspend fun createRepository(repositoryId: RepositoryId, userName: String?, useRoleIds: Boolean = true): CLVersion {
106121
var initialVersion: CLVersion? = null
107-
store.runTransaction {
122+
store.runTransactionSuspendable {
108123
val masterBranch = repositoryId.getBranchReference()
109124
if (repositoryExists(repositoryId)) throw RepositoryAlreadyExistsException(repositoryId.id)
110125
val existingRepositories = getRepositories()
@@ -160,10 +175,10 @@ class RepositoriesManager(val client: LocalModelClient) {
160175
}
161176
}
162177

163-
fun removeRepository(repository: RepositoryId): Boolean {
164-
return store.runTransaction {
178+
suspend fun removeRepository(repository: RepositoryId): Boolean {
179+
return store.runTransactionSuspendable {
165180
if (!repositoryExists(repository)) {
166-
return@runTransaction false
181+
return@runTransactionSuspendable false
167182
}
168183

169184
for (branchName in getBranchNames(repository)) {
@@ -178,9 +193,9 @@ class RepositoriesManager(val client: LocalModelClient) {
178193
}
179194
}
180195

181-
fun removeBranches(repository: RepositoryId, branchNames: Set<String>) {
196+
suspend fun removeBranches(repository: RepositoryId, branchNames: Set<String>) {
182197
if (branchNames.isEmpty()) return
183-
store.runTransaction {
198+
store.runTransactionSuspendable {
184199
val key = branchListKey(repository)
185200
val existingBranches = store[key]?.lines()?.toSet() ?: emptySet()
186201
val remainingBranches = existingBranches - branchNames
@@ -191,11 +206,10 @@ class RepositoriesManager(val client: LocalModelClient) {
191206
}
192207
}
193208

194-
fun mergeChanges(branch: BranchReference, newVersionHash: String): String {
209+
suspend fun mergeChanges(branch: BranchReference, newVersionHash: String): String {
195210
var result: String? = null
196-
store.runTransaction {
197-
val branchKey = branchKey(branch)
198-
val headHash = getVersionHash(branch)
211+
store.runTransactionSuspendable {
212+
val headHash = getVersionHashInsideTransaction(branch)
199213
val mergedHash = if (headHash == null) {
200214
newVersionHash
201215
} else {
@@ -217,17 +231,20 @@ class RepositoriesManager(val client: LocalModelClient) {
217231
return result!!
218232
}
219233

220-
fun getVersion(branch: BranchReference): CLVersion? {
234+
suspend fun getVersion(branch: BranchReference): CLVersion? {
221235
return getVersionHash(branch)?.let { CLVersion.loadFromHash(it, client.storeCache) }
222236
}
223237

224-
fun getVersionHash(branch: BranchReference): String? {
225-
return store.runTransaction {
226-
store[branchKey(branch)]
227-
?: store[legacyBranchKey(branch)]?.also { store.put(branchKey(branch), it, true) }
238+
suspend fun getVersionHash(branch: BranchReference): String? {
239+
return store.runTransactionSuspendable {
240+
getVersionHashInsideTransaction(branch)
228241
}
229242
}
230243

244+
private fun getVersionHashInsideTransaction(branch: BranchReference): String? {
245+
return store[branchKey(branch)] ?: store[legacyBranchKey(branch)]?.also { store.put(branchKey(branch), it, true) }
246+
}
247+
231248
private fun putVersionHash(branch: BranchReference, hash: String?) {
232249
store.put(branchKey(branch), hash, false)
233250
store.put(legacyBranchKey(branch), hash, false)
@@ -295,18 +312,6 @@ class RepositoriesManager(val client: LocalModelClient) {
295312

296313
private fun branchListKey(repositoryId: RepositoryId) = "$KEY_PREFIX:repositories:${repositoryId.id}:branches"
297314

298-
fun migrateLegacyRepositoriesList() {
299-
val legacyRepositories = listLegacyRepositories().groupBy { it.repositoryId }
300-
if (legacyRepositories.isNotEmpty()) {
301-
store.runTransaction {
302-
ensureRepositoriesAreInList(legacyRepositories.keys)
303-
for ((legacyRepository, legacyBranches) in legacyRepositories) {
304-
ensureBranchesAreInList(legacyRepository, legacyBranches.map { it.branchName }.toSet())
305-
}
306-
}
307-
}
308-
}
309-
310315
private fun listLegacyRepositories(): Set<BranchReference> {
311316
val result: MutableSet<BranchReference> = HashSet()
312317
val infoVersionHash = client[RepositoryId("info").getBranchReference().getKey()] ?: return emptySet()

‎model-server/src/main/kotlin/org/modelix/model/server/store/IStoreClient.kt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,11 @@
1414
*/
1515
package org.modelix.model.server.store
1616

17+
import kotlinx.coroutines.Dispatchers
1718
import kotlinx.coroutines.channels.Channel
1819
import kotlinx.coroutines.coroutineScope
1920
import kotlinx.coroutines.launch
21+
import kotlinx.coroutines.withContext
2022
import kotlinx.coroutines.withTimeoutOrNull
2123
import org.modelix.model.IKeyListener
2224
import java.io.File
@@ -36,6 +38,10 @@ interface IStoreClient : AutoCloseable {
3638
fun <T> runTransaction(body: () -> T): T
3739
}
3840

41+
suspend fun <T> IStoreClient.runTransactionSuspendable(body: () -> T): T {
42+
return withContext(Dispatchers.IO) { runTransaction(body) }
43+
}
44+
3945
suspend fun pollEntry(storeClient: IStoreClient, key: String, lastKnownValue: String?): String? {
4046
var result: String? = null
4147
coroutineScope {

‎model-server/src/main/kotlin/org/modelix/model/server/templates/PageWithMenuBar.kt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ class PageWithMenuBar(val activePage: String, val baseUrl: String) : Template<HT
6262
}
6363
div {
6464
style = "display: flex; flex-direction: column; align-items: center;"
65+
6566
insert(bodyContent)
6667
}
6768
}

‎modelql-server/src/main/kotlin/org/modelix/modelql/server/ModelQLServer.kt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ class ModelQLServer private constructor(val rootNodeProvider: () -> INode?, val
8080
handleCall(call, { rootNode to area }, {})
8181
}
8282

83-
suspend fun handleCall(call: ApplicationCall, input: suspend (write: Boolean) -> Pair<INode, IArea>, afterQueryExecution: () -> Unit = {}) {
83+
suspend fun handleCall(call: ApplicationCall, input: suspend (write: Boolean) -> Pair<INode, IArea>, afterQueryExecution: suspend () -> Unit = {}) {
8484
try {
8585
val serializedQuery = call.receiveText()
8686
val json = UntypedModelQL.json

0 commit comments

Comments
 (0)