feat(model-server): isolated storage of repository data

The versions of models where previously stored in the same storage
without knowing to which repository they belong.
To allow proper permission checks there is now an additional
`repository` field in the database.

Existing repositories remain in the global storage and don't benefit
from this new security feature. A migration is not yet implemented.

BREAKING CHANGE: New repositories are not accessible via the
    deprecated `RestWebModelClient`.
    Use IModelClientV2.initRepositoryWithLegacyStorage to create a new
    repository that is also accessible via the deprecated client.
    If you directly use the REST API to create repositories then add the
    query parameter `?legacyGlobalStorage=true`.

Author: slisson

gradle/libs.versions.toml

@@ -22,6 +22,7 @@ shadow = { id = "com.github.johnrengelman.shadow", version = "8.1.1" }
 intellij = { id = "org.jetbrains.intellij", version = "1.17.3" }
 openapi-generator = {id = "org.openapi.generator", version.ref = "openapi"}
 kotlinx-kover = { id = "org.jetbrains.kotlinx.kover", version = "0.8.0" }
+docker-compose = { id = "com.avast.gradle.docker-compose" , version = "0.17.6" }
 
 [versions]
 kotlin = "1.9.24"

kotlin-utils/src/jsMain/kotlin/org/modelix/kotlin/utils/ContextValue.js.kt

@@ -36,7 +36,9 @@ actual class ContextValue<E> {
     }
 
     actual fun getValue(): E {
-        return getAllValues().last()
+        val stack = getAllValues()
+        check(stack.isNotEmpty()) { "No value provided for ContextValue" }
+        return stack.last()
     }
 
     actual fun getValueOrNull(): E? {

kotlin-utils/src/jvmMain/kotlin/org/modelix/kotlin/utils/ContextValue.jvm.kt

@@ -41,7 +41,9 @@ actual class ContextValue<E>(private val initialStack: List<E>) {
     }
 
     actual fun getValue(): E {
-        return valueStack.get().last()
+        val stack = valueStack.get()
+        check(stack.isNotEmpty()) { "No value provided for ContextValue" }
+        return stack.last()
     }
 
     actual fun getValueOrNull(): E? {

light-model-client/src/jvmTest/kotlin/org/modelix/client/light/LightModelClientTest.kt

@@ -15,7 +15,6 @@ package org.modelix.client.light
 
 import io.ktor.client.HttpClient
 import io.ktor.client.plugins.websocket.WebSockets
-import io.ktor.client.request.post
 import io.ktor.server.application.install
 import io.ktor.server.testing.testApplication
 import kotlinx.coroutines.coroutineScope
@@ -28,8 +27,8 @@ import org.modelix.incremental.incrementalFunction
 import org.modelix.model.api.IProperty
 import org.modelix.model.api.addNewChild
 import org.modelix.model.api.getDescendants
-import org.modelix.model.server.handlers.DeprecatedLightModelServer
 import org.modelix.model.server.handlers.LightModelServer
+import org.modelix.model.server.handlers.RepositoriesManager
 import org.modelix.model.server.store.InMemoryStoreClient
 import org.modelix.model.server.store.LocalModelClient
 import org.modelix.model.test.RandomModelChangeGenerator
@@ -45,14 +44,16 @@ class LightModelClientTest {
     var localModelClient: LocalModelClient? = null
 
     private fun runTest(block: suspend (HttpClient) -> Unit) = testApplication {
+        val modelClient = LocalModelClient(InMemoryStoreClient())
+        val repositoryManager = RepositoriesManager(modelClient)
+        repositoryManager.createRepository(org.modelix.model.lazy.RepositoryId("test-repo"), userName = "unit-test")
+
         application {
             installAuthentication(unitTestMode = true)
             install(io.ktor.server.websocket.WebSockets)
             install(io.ktor.server.resources.Resources)
-            val modelClient = LocalModelClient(InMemoryStoreClient())
             localModelClient = modelClient
-            DeprecatedLightModelServer(modelClient).init(this)
-            LightModelServer(modelClient).init(this)
+            LightModelServer(modelClient, repositoryManager).init(this)
         }
         val client = createClient {
             install(WebSockets)
@@ -62,9 +63,6 @@ class LightModelClientTest {
 
     fun runClientTest(block: suspend (suspend (debugName: String) -> LightModelClient) -> Unit) = runTest { httpClient ->
         withTimeout(2.minutes) {
-            val response = httpClient.post("http://localhost/json/test-repo/init").status
-            // println("init: $response")
-
             val createClient: suspend (debugName: String) -> LightModelClient = { debugName ->
                 val client = LightModelClient.builder()
                     .httpClient(httpClient)

model-client/src/commonMain/kotlin/org/modelix/model/client2/IModelClientV2.kt

@@ -40,6 +40,7 @@ interface IModelClientV2 {
     fun getUserId(): String?
 
     suspend fun initRepository(repository: RepositoryId, useRoleIds: Boolean = true): IVersion
+    suspend fun initRepositoryWithLegacyStorage(repository: RepositoryId): IVersion
     suspend fun listRepositories(): List<RepositoryId>
     suspend fun deleteRepository(repository: RepositoryId): Boolean
     suspend fun listBranches(repository: RepositoryId): List<BranchReference>

model-client/src/commonMain/kotlin/org/modelix/model/client2/ModelClientV2.kt

@@ -16,6 +16,7 @@ package org.modelix.model.client2
 import io.ktor.client.HttpClient
 import io.ktor.client.HttpClientConfig
 import io.ktor.client.call.body
+import io.ktor.client.plugins.ClientRequestException
 import io.ktor.client.plugins.HttpRequestRetry
 import io.ktor.client.plugins.HttpTimeout
 import io.ktor.client.plugins.ResponseException
@@ -55,10 +56,12 @@ import org.modelix.model.api.INode
 import org.modelix.model.api.IdGeneratorDummy
 import org.modelix.model.api.TreePointer
 import org.modelix.model.api.getRootNode
+import org.modelix.model.api.runSynchronized
 import org.modelix.model.client.IdGenerator
 import org.modelix.model.lazy.BranchReference
 import org.modelix.model.lazy.CLTree
 import org.modelix.model.lazy.CLVersion
+import org.modelix.model.lazy.IDeserializingKeyValueStore
 import org.modelix.model.lazy.ObjectStoreCache
 import org.modelix.model.lazy.RepositoryId
 import org.modelix.model.lazy.computeDelta
@@ -74,6 +77,8 @@ import org.modelix.modelql.core.IMonoStep
 import kotlin.time.Duration
 import kotlin.time.Duration.Companion.seconds
 
+class VersionNotFoundException(val versionHash: String) : Exception("Version $versionHash not found")
+
 class ModelClientV2(
     private val httpClient: HttpClient,
     val baseUrl: String,
@@ -82,14 +87,20 @@ class ModelClientV2(
     private var clientId: Int = 0
     private var idGenerator: IIdGenerator = IdGeneratorDummy()
     private var serverProvidedUserId: String? = null
-    private val kvStore = MapBasedStore()
-    val store = ObjectStoreCache(kvStore) // TODO the store will accumulate garbage
+
+    // TODO the store will accumulate garbage
+    private val storeForRepository: MutableMap<RepositoryId?, IDeserializingKeyValueStore> = HashMap()
 
     suspend fun init() {
         updateClientId()
         updateUserId()
     }
 
+    private fun getStore(repository: RepositoryId?) = runSynchronized(storeForRepository) { storeForRepository.getOrPut(repository) { ObjectStoreCache(MapBasedStore()) } }
+    private fun getRepository(store: IDeserializingKeyValueStore): RepositoryId? {
+        return storeForRepository.asSequence().first { it.value == store }.key
+    }
+
     private suspend fun updateClientId() {
         this.clientId = httpClient.post {
             url {
@@ -127,15 +138,24 @@ class ModelClientV2(
     override fun getUserId(): String? = clientProvidedUserId ?: serverProvidedUserId
 
     override suspend fun initRepository(repository: RepositoryId, useRoleIds: Boolean): IVersion {
+        return initRepository(repository, useRoleIds = useRoleIds, legacyGlobalStorage = false)
+    }
+
+    override suspend fun initRepositoryWithLegacyStorage(repository: RepositoryId): IVersion {
+        return initRepository(repository, useRoleIds = false, legacyGlobalStorage = true)
+    }
+
+    suspend fun initRepository(repository: RepositoryId, useRoleIds: Boolean = false, legacyGlobalStorage: Boolean = false): IVersion {
         return httpClient.preparePost {
             url {
                 parameter("useRoleIds", useRoleIds)
                 takeFrom(baseUrl)
                 appendPathSegmentsEncodingSlash("repositories", repository.id, "init")
+                if (legacyGlobalStorage) parameters["legacyGlobalStorage"] = legacyGlobalStorage.toString()
             }
             useVersionStreamFormat()
         }.execute { response ->
-            createVersion(null, response.readVersionDelta())
+            createVersion(getStore(repository), null, response.readVersionDelta())
         }
     }
 
@@ -193,36 +213,55 @@ class ModelClientV2(
     @Deprecated("repository ID is required for permission checks")
     @DeprecationInfo("3.7.0", "May be removed with the next major release. Also remove the endpoint from the model-server.")
     override suspend fun loadVersion(versionHash: String, baseVersion: IVersion?): IVersion {
-        return httpClient.prepareGet {
-            url {
-                takeFrom(baseUrl)
-                appendPathSegments("versions", versionHash)
-                if (baseVersion != null) {
-                    parameters["lastKnown"] = (baseVersion as CLVersion).getContentHash()
+        val repositoryIdFromBaseVersion = (baseVersion as? CLVersion)?.let { getRepository(it.store) }
+        if (repositoryIdFromBaseVersion != null) {
+            return doLoadVersion(repositoryIdFromBaseVersion, versionHash, baseVersion)
+        } else {
+            // try finding the version in any repository
+            for (repositoryId in listRepositories() + null) {
+                try {
+                    return doLoadVersion(repositoryId, versionHash, baseVersion)
+                } catch (ex: ClientRequestException) {
+                    when (ex.response.status) {
+                        HttpStatusCode.NotFound -> {}
+                        HttpStatusCode.Unauthorized -> {}
+                        HttpStatusCode.Forbidden -> {}
+                        else -> throw ex
+                    }
                 }
             }
-            useVersionStreamFormat()
-        }.execute { response ->
-            createVersion(baseVersion as CLVersion?, response.readVersionDelta())
+            throw VersionNotFoundException(versionHash)
         }
     }
 
     override suspend fun loadVersion(
         repositoryId: RepositoryId,
         versionHash: String,
         baseVersion: IVersion?,
+    ): IVersion {
+        return doLoadVersion(repositoryId, versionHash, baseVersion)
+    }
+
+    private suspend fun doLoadVersion(
+        repositoryId: RepositoryId?,
+        versionHash: String,
+        baseVersion: IVersion?,
     ): IVersion {
         return httpClient.prepareGet {
             url {
                 takeFrom(baseUrl)
-                appendPathSegments("repositories", repositoryId.id, "versions", versionHash)
+                if (repositoryId == null) {
+                    appendPathSegments("versions", versionHash)
+                } else {
+                    appendPathSegments("repositories", repositoryId.id, "versions", versionHash)
+                }
                 if (baseVersion != null) {
                     parameters["lastKnown"] = (baseVersion as CLVersion).getContentHash()
                 }
             }
             useVersionStreamFormat()
         }.execute { response ->
-            createVersion(baseVersion as CLVersion?, response.readVersionDelta())
+            createVersion(getStore(repositoryId), baseVersion as CLVersion?, response.readVersionDelta())
         }
     }
 
@@ -249,7 +288,7 @@ class ModelClientV2(
             contentType(ContentType.Application.Json)
             setBody(delta)
         }.execute { response ->
-            createVersion(version, response.readVersionDelta())
+            createVersion(getStore(branch.repositoryId), version, response.readVersionDelta())
         }
     }
 
@@ -285,7 +324,7 @@ class ModelClientV2(
             }
             useVersionStreamFormat()
         }.execute { response ->
-            val receivedVersion = createVersion(lastKnownVersion, response.readVersionDelta())
+            val receivedVersion = createVersion(getStore(branch.repositoryId), lastKnownVersion, response.readVersionDelta())
             LOG.debug { "${clientId.toString(16)}.pull($branch, $lastKnownVersion) -> $receivedVersion" }
             receivedVersion
         }
@@ -302,7 +341,7 @@ class ModelClientV2(
         }.execute { response ->
             val receivedVersion = when (response.status) {
                 HttpStatusCode.NotFound -> null
-                HttpStatusCode.OK -> createVersion(null, response.readVersionDelta())
+                HttpStatusCode.OK -> createVersion(getStore(branch.repositoryId), null, response.readVersionDelta())
                 else -> throw ResponseException(response, response.bodyAsText())
             }
             LOG.debug { "${clientId.toString(16)}.pullIfExists($branch) -> $receivedVersion" }
@@ -352,7 +391,7 @@ class ModelClientV2(
             }
             useVersionStreamFormat()
         }.execute { response ->
-            val receivedVersion = createVersion(lastKnownVersion, response.readVersionDelta())
+            val receivedVersion = createVersion(getStore(branch.repositoryId), lastKnownVersion, response.readVersionDelta())
             LOG.debug { "${clientId.toString(16)}.poll($branch, $lastKnownVersion) -> $receivedVersion" }
             receivedVersion
         }
@@ -378,7 +417,7 @@ class ModelClientV2(
         httpClient.close()
     }
 
-    private suspend fun createVersion(baseVersion: CLVersion?, delta: VersionDeltaStream): CLVersion {
+    private suspend fun createVersion(store: IDeserializingKeyValueStore, baseVersion: CLVersion?, delta: VersionDeltaStream): CLVersion {
         delta.getObjectsAsFlow().collect {
             HashUtil.checkObjectHash(it.first, it.second)
             store.keyValueStore.put(it.first, it.second)
@@ -393,7 +432,7 @@ class ModelClientV2(
         }
     }
 
-    private suspend fun createVersion(baseVersion: CLVersion?, delta: Flow<String>): CLVersion {
+    private suspend fun createVersion(store: IDeserializingKeyValueStore, baseVersion: CLVersion?, delta: Flow<String>): CLVersion {
         var firstHash: String? = null
         var isHash = true
         var lastHash: String? = null
@@ -586,7 +625,7 @@ suspend fun <T> IModelClientV2.runWriteOnBranch(branchRef: BranchReference, body
             .takeIf { it != branchRef }
             ?.let { client.pullIfExists(it) } // master branch
         ?: client.initRepository(branchRef.repositoryId)
-    val branch = OTBranch(TreePointer(baseVersion.getTree(), client.getIdGenerator()), client.getIdGenerator(), (client as ModelClientV2).store)
+    val branch = OTBranch(TreePointer(baseVersion.getTree(), client.getIdGenerator()), client.getIdGenerator(), (baseVersion as CLVersion).store)
     val result = branch.computeWrite {
         body(branch)
     }

model-datastructure/src/commonMain/kotlin/org/modelix/model/IKeyListener.kt

@@ -15,9 +15,11 @@
 
 package org.modelix.model
 
-interface IKeyListener {
+interface IKeyListener : IGenericKeyListener<String>
+
+interface IGenericKeyListener<KeyT> {
     companion object {
         const val NULL_VALUE = "Null-nULL-NulL-nULl"
     }
-    fun changed(key: String, value: String?)
+    fun changed(key: KeyT, value: String?)
 }

model-datastructure/src/commonMain/kotlin/org/modelix/model/lazy/CLVersion.kt

@@ -253,8 +253,11 @@ class CLVersion : IVersion {
         )
 
         fun loadFromHash(hash: String, store: IDeserializingKeyValueStore): CLVersion {
-            val data = store[hash, { CPVersion.deserialize(it) }]
-                ?: throw RuntimeException("Version with hash $hash not found")
+            return tryLoadFromHash(hash, store) ?: throw RuntimeException("Version with hash $hash not found")
+        }
+
+        fun tryLoadFromHash(hash: String, store: IDeserializingKeyValueStore): CLVersion? {
+            val data = store[hash, { CPVersion.deserialize(it) }] ?: return null
             return CLVersion(data, store)
         }
     }

model-server-openapi/specifications/model-server-html.yaml

@@ -37,7 +37,7 @@ paths:
               examples:
                 Example#1:
                   value: "../repos/"
-  /content/{repository}/{branch}/latest:
+  /content/repositories/{repository}/branches/{branch}/latest:
     get:
       operationId: getContentRepositoryBranchLatest
       parameters:
@@ -54,26 +54,25 @@ paths:
       responses:
         "200":
           $ref: '#/components/responses/200'
-  /content/{versionHash}:
+  /content/repositories/{repository}/versions/{versionHash}:
+    parameters:
+      - name: repository
+        in: "path"
+        required: true
+        schema:
+          type: string
+      - name: versionHash
+        in: "path"
+        required: true
+        schema:
+          type: string
     get:
       operationId: getVersionHash
-      parameters:
-        - name: versionHash
-          in: "path"
-          required: true
-          schema:
-            type: string
       responses:
         "400":
           $ref: '#/components/responses/400'
     post:
       operationId: postVersionHash
-      parameters:
-        - name: versionHash
-          in: "path"
-          required: true
-          schema:
-            type: string
       requestBody:
         content:
           'application/json':
@@ -85,10 +84,15 @@ paths:
           $ref: '#/components/responses/400'
         "200":
           $ref: '#/components/responses/200'
-  /content/{versionHash}/{nodeId}:
+  /content/repositories/{repository}/versions/{versionHash}/{nodeId}:
     get:
       operationId: getNodeIdForVersionHash
       parameters:
+        - name: repository
+          in: "path"
+          required: true
+          schema:
+            type: string
         - name: nodeId
           in: "path"
           required: true