feat(modelql): new steps: memoize, find, findAll, sum, toFlux

The removal of the InMemoryModel reduces the memory consumption, but
increases the overhead of navigations on the model. This commit
introduces a caching mechanism for avoiding repeated traversals of the
same model parts. On average, this should improve the performance enough
to compensate the increased overhead.

Author: slisson

docs/global/modules/core/pages/howto/modelql.adoc

@@ -195,3 +195,38 @@ val html = query {
 
 `buildHtmlQuery` and the `requestFragment` operation are similar to the `buildLocalMapping` operation,
 but inside the `onSuccess` block you use the Kotlin HTML DSL.
+
+== Indices/Caching
+
+To search for a node efficiently, `.find` can be used. Internally, it creates a map of all the elements and reuses that
+in following queries.
+
+[source,kotlin]
+--
+val nodeId: String
+root.find(
+    // Provides all nodes to index
+    { it.descendants() },
+    // selects the index key for each node
+    { it.nodeReferenceAsString() },
+    // The key to search for in the current request
+    nodeId.asMono()
+)
+--
+
+It's also possible to search for multiple nodes:
+
+[source,kotlin]
+--
+val name: String
+root.findAll({ it.descendants().ofConcept(C_INamedConcept) }, { it.name }, name.asMono())
+--
+
+Internally, they both use the `memoize` operation. `memoize` stores the result of the query and reuses it without
+re-executing the query.
+
+The `find` example is equivalent to this:
+[source,kotlin]
+--
+root.memoize { it.descendants().associateBy { it.nodeReferenceAsString() } }.get(nodeId.asMono()).filterNotNull()
+--

model-api-gen-runtime/src/commonMain/kotlin/org/modelix/metamodel/GeneratedConcept.kt

@@ -208,6 +208,8 @@ class GeneratedProperty<ValueT>(
     override fun getSimpleName(): String = simpleName
 
     override fun toReference(): IPropertyReference = IPropertyReference.fromIdAndName(uid, simpleName)
+
+    override fun toString(): String = "${getUID()}(${getConcept().getShortName()}.$simpleName)"
 }
 fun IProperty.typed() = this as? ITypedProperty<*>
 
@@ -240,6 +242,8 @@ abstract class GeneratedChildLink<ChildNodeT : ITypedNode, ChildConceptT : IConc
     override fun getSimpleName(): String = simpleName
 
     override fun toReference(): IChildLinkReference = IChildLinkReference.fromIdAndName(uid, simpleName)
+
+    override fun toString(): String = "${getUID()}(${getConcept().getShortName()}.$simpleName)"
 }
 fun IChildLink.typed(): ITypedChildLink<ITypedNode> {
     return this as? ITypedChildLink<ITypedNode>
@@ -295,5 +299,7 @@ class GeneratedReferenceLink<TargetNodeT : ITypedNode, TargetConceptT : IConcept
     override fun getSimpleName(): String = simpleName
 
     override fun toReference(): IReferenceLinkReference = IReferenceLinkReference.fromIdAndName(uid, simpleName)
+
+    override fun toString(): String = "${getUID()}(${getConcept().getShortName()}.$simpleName)"
 }
 fun IReferenceLink.typed() = this as? ITypedReferenceLink<ITypedNode> ?: UnknownTypedReferenceLink(this)

model-api/src/commonMain/kotlin/org/modelix/model/api/IdGeneratorDummy.kt

@@ -4,4 +4,14 @@ class IdGeneratorDummy : IIdGenerator {
     override fun generate(): Long {
         throw UnsupportedOperationException("Unexpected generation of IDs")
     }
+
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (other == null || this::class != other::class) return false
+        return true
+    }
+
+    override fun hashCode(): Int {
+        return this::class.hashCode()
+    }
 }

model-api/src/commonMain/kotlin/org/modelix/model/api/TreeAsBranch.kt

@@ -0,0 +1,93 @@
+/*
+ * Copyright (c) 2024.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.modelix.model.api
+
+data class TreeAsBranch(override val tree: ITree) : IBranch, IReadTransaction {
+    override fun getUserObject(key: Any) = null
+
+    override fun putUserObject(key: Any, value: Any?) {
+        throw UnsupportedOperationException("read-only")
+    }
+
+    override fun addListener(l: IBranchListener) {
+        // branch will never change
+    }
+
+    override fun getId(): String {
+        return tree.getId() ?: throw UnsupportedOperationException()
+    }
+
+    override fun runRead(runnable: () -> Unit) {
+        computeRead(runnable)
+    }
+
+    override fun <T> computeRead(computable: () -> T): T {
+        return RoleAccessContext.runWith(tree.usesRoleIds()) { computable() }
+    }
+
+    override fun runWrite(runnable: () -> Unit) {
+        throw UnsupportedOperationException("read-only")
+    }
+
+    override fun <T> computeWrite(computable: () -> T): T {
+        throw UnsupportedOperationException("read-only")
+    }
+
+    override fun canRead(): Boolean {
+        return true
+    }
+
+    override fun canWrite(): Boolean {
+        return false
+    }
+
+    override val transaction: ITransaction
+        get() = this
+    override val readTransaction: IReadTransaction
+        get() = this
+    override val writeTransaction: IWriteTransaction
+        get() = throw UnsupportedOperationException("read-only")
+
+    override fun removeListener(l: IBranchListener) {
+        // branch will never change
+    }
+
+    override val branch: IBranch
+        get() = this
+
+    override fun containsNode(nodeId: Long) = tree.containsNode(nodeId)
+
+    override fun getConcept(nodeId: Long) = tree.getConcept(nodeId)
+
+    override fun getConceptReference(nodeId: Long): IConceptReference? = tree.getConceptReference(nodeId)
+
+    override fun getParent(nodeId: Long) = tree.getParent(nodeId)
+
+    override fun getRole(nodeId: Long) = tree.getRole(nodeId)
+
+    override fun getProperty(nodeId: Long, role: String) = tree.getProperty(nodeId, role)
+
+    override fun getReferenceTarget(sourceId: Long, role: String) = tree.getReferenceTarget(sourceId, role)
+
+    override fun getChildren(parentId: Long, role: String?) = tree.getChildren(parentId, role)
+
+    override fun getAllChildren(parentId: Long) = tree.getAllChildren(parentId)
+
+    override fun getReferenceRoles(sourceId: Long) = tree.getReferenceRoles(sourceId)
+
+    override fun getPropertyRoles(sourceId: Long) = tree.getPropertyRoles(sourceId)
+}

model-datastructure/src/commonMain/kotlin/org/modelix/model/lazy/CLTree.kt

@@ -133,6 +133,19 @@ class CLTree(val data: CPTree, val asyncStore: IAsyncObjectStore) : ITree by Asy
         return "CLTree[${data.hash}]"
     }
 
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (other == null || this::class != other::class) return false
+
+        other as CLTree
+
+        return hash == other.hash
+    }
+
+    override fun hashCode(): Int {
+        return hash.hashCode()
+    }
+
     companion object {
         fun builder(store: IDeserializingKeyValueStore) = Builder(store.getAsyncStore())
         fun builder(store: IAsyncObjectStore) = Builder(store)

model-datastructure/src/commonMain/kotlin/org/modelix/model/lazy/ObjectStoreCache.kt

@@ -174,12 +174,14 @@ class ObjectStoreCache @JvmOverloads constructor(
 class LRUCache<K : Any, V>(val maxSize: Int) {
     private val map: MutableMap<K, V> = LinkedHashMap()
 
+    @Synchronized
     operator fun set(key: K, value: V) {
         map.remove(key)
         map[key] = value
         while (map.size > maxSize) map.remove(map.iterator().next().key)
     }
 
+    @Synchronized
     operator fun get(key: K, updatePosition: Boolean = true): V? {
         return map[key]?.also { value ->
             if (updatePosition) {
@@ -189,10 +191,12 @@ class LRUCache<K : Any, V>(val maxSize: Int) {
         }
     }
 
+    @Synchronized
     fun remove(key: K) {
         map.remove(key)
     }
 
+    @Synchronized
     fun clear() {
         map.clear()
     }

model-server/src/main/kotlin/org/modelix/model/server/handlers/ModelReplicationServer.kt

@@ -14,6 +14,7 @@
 
 package org.modelix.model.server.handlers
 
+import com.google.common.cache.CacheBuilder
 import io.ktor.http.ContentType
 import io.ktor.http.HttpStatusCode
 import io.ktor.server.application.Application
@@ -43,9 +44,12 @@ import org.modelix.authorization.checkPermission
 import org.modelix.authorization.getUserName
 import org.modelix.authorization.hasPermission
 import org.modelix.authorization.requiresLogin
+import org.modelix.model.api.IBranch
 import org.modelix.model.api.PBranch
+import org.modelix.model.api.TreeAsBranch
 import org.modelix.model.api.TreePointer
 import org.modelix.model.api.getRootNode
+import org.modelix.model.api.runSynchronized
 import org.modelix.model.area.getArea
 import org.modelix.model.client2.checkObjectHashes
 import org.modelix.model.client2.getAllObjects
@@ -59,7 +63,15 @@ import org.modelix.model.server.api.v2.VersionDelta
 import org.modelix.model.server.api.v2.VersionDeltaStream
 import org.modelix.model.server.api.v2.VersionDeltaStreamV2
 import org.modelix.model.server.store.StoreManager
+import org.modelix.modelql.core.IMemoizationPersistence
+import org.modelix.modelql.core.IStepOutput
+import org.modelix.modelql.core.MonoUnboundQuery
+import org.modelix.modelql.core.QueryEvaluationContext
+import org.modelix.modelql.core.QueryGraphDescriptor
+import org.modelix.modelql.core.upcast
 import org.modelix.modelql.server.ModelQLServer
+import org.modelix.streams.exactlyOne
+import org.modelix.streams.getSynchronous
 import org.slf4j.LoggerFactory
 
 /**
@@ -76,6 +88,7 @@ class ModelReplicationServer(
     }
 
     private val stores: StoreManager get() = repositoriesManager.getStoreManager()
+    private val indexPersistence: IMemoizationPersistence = InMemoryMemoizationPersistence()
 
     fun init(application: Application) {
         application.routing {
@@ -257,36 +270,45 @@ class ModelReplicationServer(
 
     override suspend fun PipelineContext<Unit, ApplicationCall>.postRepositoryBranchQuery(
         repository: String,
-        branch: String,
+        branchName: String,
     ) {
-        val branchRef = repositoryId(repository).getBranchReference(branch)
+        val branchRef = repositoryId(repository).getBranchReference(branchName)
         checkPermission(ModelServerPermissionSchema.branch(branchRef).query)
         val version = repositoriesManager.getVersion(branchRef) ?: throw BranchNotFoundException(branchRef)
         LOG.trace("Running query on {} @ {}", branchRef, version)
         val initialTree = version.getTree()
-        val otBranch = OTBranch(
-            PBranch(initialTree, stores.idGenerator),
-            stores.idGenerator,
-            repositoriesManager.getLegacyObjectStore(RepositoryId(repository)),
-        )
 
-        ModelQLServer.handleCall(call, { writeAccess ->
-            otBranch.getRootNode() to otBranch.getArea()
-        }, {
-            // writing the new version has to happen before call.respond is invoked, otherwise subsequent queries
-            // from the same client may still be executed on the old version.
-            val (ops, newTree) = otBranch.getPendingChanges()
-            if (newTree != initialTree) {
-                val newVersion = CLVersion.createRegularVersion(
-                    id = stores.idGenerator.generate(),
-                    author = getUserName(),
-                    tree = newTree,
-                    baseVersion = version,
-                    operations = ops.map { it.getOriginalOp() }.toTypedArray(),
-                )
-                repositoriesManager.mergeChanges(branchRef, newVersion.getContentHash())
-            }
-        })
+        IMemoizationPersistence.CONTEXT_INSTANCE.runInCoroutine(indexPersistence) {
+            lateinit var branch: IBranch
+            ModelQLServer.handleCall(call, { writeAccess ->
+                branch = if (writeAccess) {
+                    OTBranch(
+                        PBranch(initialTree, stores.idGenerator),
+                        stores.idGenerator,
+                        repositoriesManager.getLegacyObjectStore(RepositoryId(repository)),
+                    )
+                } else {
+                    TreeAsBranch(initialTree)
+                }
+                branch.getRootNode() to branch.getArea()
+            }, {
+                // writing the new version has to happen before call.respond is invoked, otherwise subsequent queries
+                // from the same client may still be executed on the old version.
+                (branch as? OTBranch)?.let { otBranch ->
+                    val (ops, newTree) = otBranch.getPendingChanges()
+                    if (newTree != initialTree) {
+                        val newVersion = CLVersion.createRegularVersion(
+                            id = stores.idGenerator.generate(),
+                            author = getUserName(),
+                            tree = newTree,
+                            baseVersion = version,
+                            operations = ops.map { it.getOriginalOp() }.toTypedArray(),
+                        )
+                        repositoriesManager.mergeChanges(branchRef, newVersion.getContentHash())
+                    }
+                }
+            })
+        }
     }
 
     override suspend fun PipelineContext<Unit, ApplicationCall>.postRepositoryVersionHashQuery(
@@ -445,3 +467,35 @@ private fun PipelineContext<Unit, ApplicationCall>.parameter(name: String): Stri
 private fun ApplicationCall.parameter(name: String): String {
     return requireNotNull(parameters[name]) { "Unknown parameter '$name'" }
 }
+
+class InMemoryMemoizationPersistence : IMemoizationPersistence {
+
+    private val cache = CacheBuilder.newBuilder().softValues().build<IndexCacheKey, IStepOutput<*>>()
+
+    /**
+     * Used for deduplication of instances to safe memory.
+     */
+    private val descriptorInstances = CacheBuilder.newBuilder().maximumSize(100).build<QueryGraphDescriptor, QueryGraphDescriptor>()
+
+    override fun <In, Out> getMemoizer(query: MonoUnboundQuery<In, Out>): IMemoizationPersistence.Memoizer<In, Out> {
+        return MemoizerImpl(query, query.createDescriptor().normalize().deduplicate())
+    }
+
+    private inner class MemoizerImpl<In, Out>(val query: MonoUnboundQuery<In, Out>, val normalizedQueryDescriptor: QueryGraphDescriptor) : IMemoizationPersistence.Memoizer<In, Out> {
+        override fun memoize(input: IStepOutput<In>): IStepOutput<Out> {
+            runSynchronized(cache) {
+                return cache.get(IndexCacheKey(normalizedQueryDescriptor, input)) {
+                    query.asFlow(QueryEvaluationContext.EMPTY, input).exactlyOne().getSynchronous()
+                }.upcast()
+            }
+        }
+    }
+
+    private fun QueryGraphDescriptor.deduplicate() = descriptorInstances.get(this) { this }
+
+    private data class IndexCacheKey(val query: QueryGraphDescriptor, val input: Any?)
+
+    private class IndexData<K, V>(val map: Map<K, List<IStepOutput<V>>>)
+
+    private fun <K, V> IndexData<*, *>.upcast() = this as IndexData<K, V>
+}

model-server/src/main/kotlin/org/modelix/model/server/store/StoreManager.kt

@@ -49,7 +49,12 @@ class StoreManager(val genericStore: IsolatingStore) {
         val existing = repositorySpecificStores[repository]?.get()
         if (existing != null) return existing
 
-        val newStore = BulkAsyncStore(CachingAsyncStore(StoreClientAsAsyncStore(getStoreClient(repository))))
+        val newStore = BulkAsyncStore(
+            CachingAsyncStore(
+                StoreClientAsAsyncStore(getStoreClient(repository)),
+                cacheSize = System.getenv("MODELIX_OBJECT_CACHE_SIZE")?.toIntOrNull() ?: 500_000,
+            ),
+        )
         repositorySpecificStores[repository] = SoftReference(newStore)
         return newStore
     }