refactor(model-server): extract IRepositoriesManager interface to make route handlers testable

Author: Oleksandr Dzhychko

model-server/src/main/kotlin/org/modelix/model/server/Main.kt

@@ -48,6 +48,7 @@ import org.apache.commons.io.FileUtils
 import org.apache.ignite.Ignition
 import org.modelix.authorization.KeycloakUtils
 import org.modelix.authorization.installAuthentication
+import org.modelix.model.InMemoryModels
 import org.modelix.model.server.handlers.ContentExplorer
 import org.modelix.model.server.handlers.DeprecatedLightModelServer
 import org.modelix.model.server.handlers.HistoryHandler
@@ -150,8 +151,9 @@ object Main {
                 i += 2
             }
             val localModelClient = LocalModelClient(storeClient)
+            val inMemoryModels = InMemoryModels()
             val repositoriesManager = RepositoriesManager(localModelClient)
-            val modelServer = KeyValueLikeModelServer(repositoriesManager)
+            val modelServer = KeyValueLikeModelServer(repositoriesManager, storeClient, inMemoryModels)
             val sharedSecretFile = cmdLineArgs.secretFile
             if (sharedSecretFile.exists()) {
                 modelServer.setSharedSecret(
@@ -162,7 +164,7 @@ object Main {
             val repositoryOverview = RepositoryOverview(repositoriesManager)
             val historyHandler = HistoryHandler(localModelClient, repositoriesManager)
             val contentExplorer = ContentExplorer(localModelClient, repositoriesManager)
-            val modelReplicationServer = ModelReplicationServer(repositoriesManager)
+            val modelReplicationServer = ModelReplicationServer(repositoriesManager, localModelClient, inMemoryModels)
             val metricsHandler = MetricsHandler()
 
             val configureNetty: NettyApplicationEngine.Configuration.() -> Unit = {

model-server/src/main/kotlin/org/modelix/model/server/handlers/ContentExplorer.kt

@@ -50,7 +50,7 @@ import org.modelix.model.lazy.RepositoryId
 import org.modelix.model.server.templates.PageWithMenuBar
 import kotlin.collections.set
 
-class ContentExplorer(private val client: IModelClient, private val repoManager: RepositoriesManager) {
+class ContentExplorer(private val client: IModelClient, private val repoManager: IRepositoriesManager) {
 
     fun init(application: Application) {
         application.routing {

model-server/src/main/kotlin/org/modelix/model/server/handlers/HistoryHandler.kt

@@ -55,7 +55,7 @@ import org.modelix.model.server.templates.PageWithMenuBar
 import java.time.LocalDateTime
 import java.time.format.DateTimeFormatter
 
-class HistoryHandler(val client: IModelClient, private val repositoriesManager: RepositoriesManager) {
+class HistoryHandler(val client: IModelClient, private val repositoriesManager: IRepositoriesManager) {
 
     fun init(application: Application) {
         application.routing {

model-server/src/main/kotlin/org/modelix/model/server/handlers/IRepositoriesManager.kt

@@ -0,0 +1,62 @@
+/*
+ * Copyright (c) 2024.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.modelix.model.server.handlers
+
+import org.modelix.model.lazy.BranchReference
+import org.modelix.model.lazy.CLVersion
+import org.modelix.model.lazy.RepositoryId
+
+interface IRepositoriesManager {
+    /**
+     * Used to retrieve the server ID. If needed, the server ID is created and stored.
+     *
+     * If a server ID was not created yet, it is generated and saved in the database.
+     * It gets stored under the current and all legacy database keys.
+     *
+     * If the server ID was created previously but is only stored under a legacy database key,
+     * it also gets stored under the current and all legacy database keys.
+     */
+    suspend fun maybeInitAndGetSeverId(): String
+    fun getRepositories(): Set<RepositoryId>
+    suspend fun createRepository(repositoryId: RepositoryId, userName: String?, useRoleIds: Boolean = true): CLVersion
+    suspend fun removeRepository(repository: RepositoryId): Boolean
+
+    fun getBranches(repositoryId: RepositoryId): Set<BranchReference>
+
+    suspend fun removeBranches(repository: RepositoryId, branchNames: Set<String>)
+
+    /**
+     * Same as [removeBranches] but blocking.
+     * Caller is expected to execute it outside the request thread.
+     */
+    fun removeBranchesBlocking(repository: RepositoryId, branchNames: Set<String>)
+    suspend fun getVersion(branch: BranchReference): CLVersion?
+    suspend fun getVersionHash(branch: BranchReference): String?
+    suspend fun pollVersionHash(branch: BranchReference, lastKnown: String?): String
+    suspend fun mergeChanges(branch: BranchReference, newVersionHash: String): String
+
+    /**
+     * Same as [mergeChanges] but blocking.
+     * Caller is expected to execute it outside the request thread.
+     */
+    fun mergeChangesBlocking(branch: BranchReference, newVersionHash: String): String
+    suspend fun computeDelta(versionHash: String, baseVersionHash: String?): ObjectData
+}
+
+fun IRepositoriesManager.getBranchNames(repositoryId: RepositoryId): Set<String> {
+    return getBranches(repositoryId).map { it.branchName }.toSet()
+}

model-server/src/main/kotlin/org/modelix/model/server/handlers/KeyValueLikeModelServer.kt

@@ -44,6 +44,7 @@ import org.modelix.authorization.checkPermission
 import org.modelix.authorization.getUserName
 import org.modelix.authorization.requiresPermission
 import org.modelix.authorization.toKeycloakScope
+import org.modelix.model.InMemoryModels
 import org.modelix.model.lazy.BranchReference
 import org.modelix.model.lazy.RepositoryId
 import org.modelix.model.persistent.HashUtil
@@ -67,16 +68,22 @@ private class NotFoundException(description: String?) : RuntimeException(descrip
 
 typealias CallContext = PipelineContext<Unit, ApplicationCall>
 
-class KeyValueLikeModelServer(val repositoriesManager: RepositoriesManager) {
+class KeyValueLikeModelServer(
+    private val repositoriesManager: IRepositoriesManager,
+    private val storeClient: IStoreClient,
+    private val inMemoryModels: InMemoryModels,
+) {
+
+    constructor(repositoriesManager: RepositoriesManager) :
+        this(repositoriesManager, repositoriesManager.client.store, InMemoryModels())
+
     companion object {
         private val LOG = LoggerFactory.getLogger(KeyValueLikeModelServer::class.java)
         val HASH_PATTERN = Pattern.compile("[a-zA-Z0-9\\-_]{5}\\*[a-zA-Z0-9\\-_]{38}")
         const val PROTECTED_PREFIX = "$$$"
         val HEALTH_KEY = PROTECTED_PREFIX + "health2"
     }
 
-    val storeClient: IStoreClient get() = repositoriesManager.client.store
-
     fun init(application: Application) {
         // Functionally, it does not matter if the server ID
         // is created eagerly on startup or lazily on the first request,
@@ -100,7 +107,7 @@ class KeyValueLikeModelServer(val repositoriesManager: RepositoriesManager) {
                     ?.getBranchReference(System.getenv("MODELIX_SERVER_MODELQL_WARMUP_BRANCH"))
                 if (branchRef != null) {
                     val version = repositoriesManager.getVersion(branchRef)
-                    if (repositoriesManager.inMemoryModels.getModel(version!!.getTree()).isActive) {
+                    if (inMemoryModels.getModel(version!!.getTree()).isActive) {
                         call.respondText(
                             status = HttpStatusCode.ServiceUnavailable,
                             text = "Waiting for version $version to be loaded into memory",
@@ -346,7 +353,7 @@ class KeyValueLikeModelServer(val repositoriesManager: RepositoriesManager) {
 
         HashUtil.checkObjectHashes(hashedObjects)
 
-        repositoriesManager.client.store.runTransactionSuspendable {
+        storeClient.runTransactionSuspendable {
             storeClient.putAll(hashedObjects)
             storeClient.putAll(userDefinedEntries)
             for ((branch, value) in branchChanges) {

model-server/src/main/kotlin/org/modelix/model/server/handlers/ModelReplicationServer.kt

@@ -46,6 +46,7 @@ import kotlinx.serialization.encodeToString
 import kotlinx.serialization.json.Json
 import org.modelix.api.public.Paths
 import org.modelix.authorization.getUserName
+import org.modelix.model.InMemoryModels
 import org.modelix.model.api.ITree
 import org.modelix.model.api.PBranch
 import org.modelix.model.api.TreePointer
@@ -69,15 +70,20 @@ import org.slf4j.LoggerFactory
  * Implements the endpoints used by the 'model-client', but compared to KeyValueLikeModelServer also understands what
  * client sends. This allows more validations and more responsibilities on the server side.
  */
-class ModelReplicationServer(val repositoriesManager: RepositoriesManager) {
-    constructor(modelClient: LocalModelClient) : this(RepositoriesManager(modelClient))
+class ModelReplicationServer(
+    private val repositoriesManager: IRepositoriesManager,
+    private val modelClient: LocalModelClient,
+    private val inMemoryModels: InMemoryModels,
+) {
+    constructor(repositoriesManager: RepositoriesManager) :
+        this(repositoriesManager, repositoriesManager.client, InMemoryModels())
+
+    constructor(modelClient: LocalModelClient) : this(RepositoriesManager(modelClient), modelClient, InMemoryModels())
     constructor(storeClient: IStoreClient) : this(LocalModelClient(storeClient))
 
     companion object {
         private val LOG = LoggerFactory.getLogger(ModelReplicationServer::class.java)
     }
-
-    private val modelClient: LocalModelClient get() = repositoriesManager.client
     private val storeClient: IStoreClient get() = modelClient.store
 
     fun init(application: Application) {
@@ -268,16 +274,16 @@ class ModelReplicationServer(val repositoriesManager: RepositoriesManager) {
             LOG.trace("Running query on {} @ {}", branchRef, version)
             val initialTree = version!!.getTree()
             val branch = OTBranch(
-                PBranch(initialTree, repositoriesManager.client.idGenerator),
-                repositoriesManager.client.idGenerator,
-                repositoriesManager.client.storeCache,
+                PBranch(initialTree, modelClient.idGenerator),
+                modelClient.idGenerator,
+                modelClient.storeCache,
             )
 
             ModelQLServer.handleCall(call, { writeAccess ->
                 if (writeAccess) {
                     branch.getRootNode() to branch.getArea()
                 } else {
-                    val model = repositoriesManager.inMemoryModels.getModel(initialTree).await()
+                    val model = inMemoryModels.getModel(initialTree).await()
                     model.getNode(ITree.ROOT_ID) to model.getArea()
                 }
             }, {
@@ -286,7 +292,7 @@ class ModelReplicationServer(val repositoriesManager: RepositoriesManager) {
                 val (ops, newTree) = branch.getPendingChanges()
                 if (newTree != initialTree) {
                     val newVersion = CLVersion.createRegularVersion(
-                        id = repositoriesManager.client.idGenerator.generate(),
+                        id = modelClient.idGenerator.generate(),
                         author = getUserName(),
                         tree = newTree as CLTree,
                         baseVersion = version,
@@ -299,7 +305,7 @@ class ModelReplicationServer(val repositoriesManager: RepositoriesManager) {
 
         post<Paths.postRepositoryVersionHashQuery> {
             val versionHash = call.parameters["versionHash"]!!
-            val version = CLVersion.loadFromHash(versionHash, repositoriesManager.client.storeCache)
+            val version = CLVersion.loadFromHash(versionHash, modelClient.storeCache)
             val initialTree = version.getTree()
             val branch = TreePointer(initialTree)
             ModelQLServer.handleCall(call, branch.getRootNode(), branch.getArea())

model-server/src/main/kotlin/org/modelix/model/server/handlers/RepositoriesManager.kt

@@ -25,7 +25,6 @@ import kotlinx.coroutines.launch
 import kotlinx.coroutines.runBlocking
 import kotlinx.datetime.Clock
 import org.apache.commons.collections4.map.LRUMap
-import org.modelix.model.InMemoryModels
 import org.modelix.model.ModelMigrations
 import org.modelix.model.VersionMerger
 import org.modelix.model.api.IBranch
@@ -51,7 +50,7 @@ import org.slf4j.LoggerFactory
 import java.lang.ref.SoftReference
 import java.util.UUID
 
-class RepositoriesManager(val client: LocalModelClient) {
+class RepositoriesManager(val client: LocalModelClient) : IRepositoriesManager {
 
     init {
         fun migrateLegacyRepositoriesList(infoBranch: IBranch) {
@@ -82,13 +81,6 @@ class RepositoriesManager(val client: LocalModelClient) {
 
     private val store: IStoreClient get() = client.store
     private val objectStore: IDeserializingKeyValueStore get() = client.storeCache
-    val inMemoryModels = InMemoryModels()
-
-    fun dispose() {
-        // TODO find instance creations and add a dispose() call if needed. Whoever creates an instance is responsible
-        //      for its lifecycle.
-        inMemoryModels.dispose()
-    }
 
     fun generateClientId(repositoryId: RepositoryId): Long {
         return client.store.generateId("$KEY_PREFIX:${repositoryId.id}:clientId")
@@ -103,7 +95,7 @@ class RepositoriesManager(val client: LocalModelClient) {
      * If the server ID was created previously but is only stored under a legacy database key,
      * it also gets stored under the current and all legacy database keys.
      */
-    suspend fun maybeInitAndGetSeverId(): String {
+    override suspend fun maybeInitAndGetSeverId(): String {
         return store.runTransactionSuspendable {
             var serverId = store[SERVER_ID_KEY]
             if (serverId == null) {
@@ -118,7 +110,7 @@ class RepositoriesManager(val client: LocalModelClient) {
         }
     }
 
-    fun getRepositories(): Set<RepositoryId> {
+    override fun getRepositories(): Set<RepositoryId> {
         val repositoriesList = store[REPOSITORIES_LIST_KEY]
         val emptyRepositoriesList = repositoriesList.isNullOrBlank()
         return if (emptyRepositoriesList) {
@@ -130,7 +122,7 @@ class RepositoriesManager(val client: LocalModelClient) {
 
     private fun repositoryExists(repositoryId: RepositoryId) = getRepositories().contains(repositoryId)
 
-    suspend fun createRepository(repositoryId: RepositoryId, userName: String?, useRoleIds: Boolean = true): CLVersion {
+    override suspend fun createRepository(repositoryId: RepositoryId, userName: String?, useRoleIds: Boolean): CLVersion {
         var initialVersion: CLVersion? = null
         store.runTransactionSuspendable {
             val masterBranch = repositoryId.getBranchReference()
@@ -155,7 +147,7 @@ class RepositoriesManager(val client: LocalModelClient) {
         return store[branchListKey(repositoryId)]?.lines()?.toSet() ?: emptySet()
     }
 
-    fun getBranches(repositoryId: RepositoryId): Set<BranchReference> {
+    override fun getBranches(repositoryId: RepositoryId): Set<BranchReference> {
         return getBranchNames(repositoryId)
             .map { repositoryId.getBranchReference(it) }
             .sortedBy { it.branchName }
@@ -188,7 +180,7 @@ class RepositoriesManager(val client: LocalModelClient) {
         }
     }
 
-    suspend fun removeRepository(repository: RepositoryId): Boolean {
+    override suspend fun removeRepository(repository: RepositoryId): Boolean {
         return store.runTransactionSuspendable {
             if (!repositoryExists(repository)) {
                 return@runTransactionSuspendable false
@@ -206,7 +198,7 @@ class RepositoriesManager(val client: LocalModelClient) {
         }
     }
 
-    suspend fun removeBranches(repository: RepositoryId, branchNames: Set<String>) {
+    override suspend fun removeBranches(repository: RepositoryId, branchNames: Set<String>) {
         return store.runTransactionSuspendable {
             removeBranchesBlocking(repository, branchNames)
         }
@@ -216,7 +208,7 @@ class RepositoriesManager(val client: LocalModelClient) {
      * Same as [removeBranches] but blocking.
      * Caller is expected to execute it outside the request thread.
      */
-    fun removeBranchesBlocking(repository: RepositoryId, branchNames: Set<String>) {
+    override fun removeBranchesBlocking(repository: RepositoryId, branchNames: Set<String>) {
         if (branchNames.isEmpty()) return
         store.runTransaction {
             val key = branchListKey(repository)
@@ -229,7 +221,7 @@ class RepositoriesManager(val client: LocalModelClient) {
         }
     }
 
-    suspend fun mergeChanges(branch: BranchReference, newVersionHash: String): String {
+    override suspend fun mergeChanges(branch: BranchReference, newVersionHash: String): String {
         return store.runTransactionSuspendable {
             mergeChangesBlocking(branch, newVersionHash)
         }
@@ -239,7 +231,7 @@ class RepositoriesManager(val client: LocalModelClient) {
      * Same as [mergeChanges] but blocking.
      * Caller is expected to execute it outside the request thread.
      */
-    fun mergeChangesBlocking(branch: BranchReference, newVersionHash: String): String {
+    override fun mergeChangesBlocking(branch: BranchReference, newVersionHash: String): String {
         return store.runTransaction {
             val headHash = getVersionHashBlocking(branch)
             val mergedHash = if (headHash == null) {
@@ -262,11 +254,11 @@ class RepositoriesManager(val client: LocalModelClient) {
         }
     }
 
-    suspend fun getVersion(branch: BranchReference): CLVersion? {
+    override suspend fun getVersion(branch: BranchReference): CLVersion? {
         return getVersionHash(branch)?.let { CLVersion.loadFromHash(it, client.storeCache) }
     }
 
-    suspend fun getVersionHash(branch: BranchReference): String? {
+    override suspend fun getVersionHash(branch: BranchReference): String? {
         return store.runTransactionSuspendable {
             getVersionHashBlocking(branch)
         }
@@ -293,13 +285,13 @@ class RepositoriesManager(val client: LocalModelClient) {
         store.put(legacyBranchKey(branch), hash, false)
     }
 
-    suspend fun pollVersionHash(branch: BranchReference, lastKnown: String?): String {
+    override suspend fun pollVersionHash(branch: BranchReference, lastKnown: String?): String {
         return pollEntry(client.store, branchKey(branch), lastKnown)
             ?: throw IllegalStateException("No version found for branch '${branch.branchName}' in repository '${branch.repositoryId}'")
     }
 
     private val versionDeltaCache = VersionDeltaCache(client.storeCache)
-    suspend fun computeDelta(versionHash: String, baseVersionHash: String?): ObjectData {
+    override suspend fun computeDelta(versionHash: String, baseVersionHash: String?): ObjectData {
         if (versionHash == baseVersionHash) return ObjectData.empty
         if (baseVersionHash == null) {
             // no need to cache anything if there is no delta computation happening

model-server/src/main/kotlin/org/modelix/model/server/handlers/RepositoryOverview.kt

@@ -25,7 +25,7 @@ import kotlinx.html.tr
 import org.modelix.api.html.Paths
 import org.modelix.model.server.templates.PageWithMenuBar
 
-class RepositoryOverview(private val repoManager: RepositoriesManager) {
+class RepositoryOverview(private val repoManager: IRepositoriesManager) {
 
     fun init(application: Application) {
         application.routing {