feat(model-datastructure): new BulkUpdateOp to merge large updates into a single operation

A bulk import usually generates a lot of change operations, but an operation should capture the
intended change as high level as possible. In case of a bulk import the intention is to put the
model into the resulting state, which can be captured more accurately and more memory efficient with
the new operation.

Author: slisson

bulk-model-sync-gradle/src/main/kotlin/org/modelix/model/sync/bulk/gradle/tasks/ImportIntoModelServer.kt

@@ -30,9 +30,12 @@ import org.gradle.api.tasks.TaskAction
 import org.modelix.model.ModelFacade
 import org.modelix.model.api.ILanguage
 import org.modelix.model.api.ILanguageRepository
+import org.modelix.model.api.INode
+import org.modelix.model.api.PNodeAdapter
 import org.modelix.model.client2.ModelClientV2
 import org.modelix.model.client2.runWrite
 import org.modelix.model.lazy.RepositoryId
+import org.modelix.model.operations.OTBranch
 import org.modelix.model.sync.bulk.ModelImporter
 import org.modelix.model.sync.bulk.importFilesAsRootChildren
 import org.modelix.model.sync.bulk.isModuleIncluded
@@ -84,11 +87,20 @@ abstract class ImportIntoModelServer @Inject constructor(of: ObjectFactory) : De
         runBlocking {
             client.init()
             client.runWrite(branchRef) { rootNode ->
-                logger.info("Got root node: {}", rootNode)
-                logger.info("Importing...")
-                ModelImporter(rootNode, continueOnError.get()).importFilesAsRootChildren(files)
-                logger.info("Import finished")
+                rootNode.runBulkUpdate {
+                    logger.info("Got root node: {}", rootNode)
+                    logger.info("Importing...")
+                    ModelImporter(rootNode, continueOnError.get()).importFilesAsRootChildren(files)
+                    logger.info("Import finished")
+                }
             }
         }
     }
 }
+
+/**
+ * Memory optimization that doesn't record individual change operations, but only the result.
+ */
+private fun INode.runBulkUpdate(body: () -> Unit) {
+    ((this as PNodeAdapter).branch as OTBranch).runBulkUpdate(body = body)
+}

model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/BulkUpdateOp.kt

@@ -0,0 +1,82 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.modelix.model.operations
+
+import org.modelix.model.api.ITree
+import org.modelix.model.api.IWriteTransaction
+import org.modelix.model.lazy.CLTree
+import org.modelix.model.lazy.IDeserializingKeyValueStore
+import org.modelix.model.lazy.KVEntryReference
+import org.modelix.model.persistent.CPTree
+import org.modelix.model.persistent.IKVValue
+import org.modelix.model.persistent.SerializationUtil
+
+class BulkUpdateOp(
+    val resultTreeHash: KVEntryReference<CPTree>,
+    val subtreeRootId: Long,
+) : AbstractOperation() {
+
+    override fun getReferencedEntries(): List<KVEntryReference<IKVValue>> = listOf(resultTreeHash)
+
+    /**
+     * Since this operation is recorded at the end of a bulk update we need to create an IAppliedOperation without
+     * actually applying it again.
+     */
+    fun afterApply(baseTree: CLTree) = Applied(baseTree)
+
+    override fun apply(transaction: IWriteTransaction, store: IDeserializingKeyValueStore): IAppliedOperation {
+        val baseTree = transaction.tree as CLTree
+        val resultTree = getResultTree(store)
+        TODO("Change the (sub)tree so that it is identical to the resultTree")
+        return Applied(baseTree)
+    }
+
+    private fun getResultTree(store: IDeserializingKeyValueStore): CLTree = CLTree(resultTreeHash.getValue(store), store)
+
+    override fun toString(): String {
+        return "BulkUpdateOp ${resultTreeHash.getHash()}, ${SerializationUtil.longToHex(subtreeRootId)}"
+    }
+
+    override fun toCode(): String {
+        return """// TODO BulkUpdateOp"""
+    }
+
+    inner class Applied(val baseTree: CLTree) : AbstractOperation.Applied(), IAppliedOperation {
+        override fun getOriginalOp() = this@BulkUpdateOp
+
+        override fun invert(): List<IOperation> {
+            return listOf(BulkUpdateOp(KVEntryReference(baseTree.data), subtreeRootId))
+        }
+    }
+
+    override fun captureIntend(tree: ITree, store: IDeserializingKeyValueStore): IOperationIntend {
+        return Intend()
+    }
+
+    inner class Intend : IOperationIntend {
+        override fun restoreIntend(tree: ITree): List<IOperation> {
+            // The intended change is to put the model into the given state. Any concurrent change can just be
+            // overwritten as long as the subtree root as the starting point still exists.
+            return if (tree.containsNode(subtreeRootId)) {
+                listOf(getOriginalOp())
+            } else {
+                listOf(NoOp())
+            }
+        }
+
+        override fun getOriginalOp() = this@BulkUpdateOp
+    }
+}

model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/OTBranch.kt

@@ -23,20 +23,48 @@ import org.modelix.model.api.ITransaction
 import org.modelix.model.api.ITree
 import org.modelix.model.api.IWriteTransaction
 import org.modelix.model.api.runSynchronized
+import org.modelix.model.lazy.CLTree
 import org.modelix.model.lazy.IDeserializingKeyValueStore
+import org.modelix.model.lazy.KVEntryReference
 
 class OTBranch(
     private val branch: IBranch,
     private val idGenerator: IIdGenerator,
     private val store: IDeserializingKeyValueStore,
 ) : IBranch {
+    private var bulkUpdateMode: Boolean = false
     private var currentOperations: MutableList<IAppliedOperation> = ArrayList()
     private val completedChanges: MutableList<OpsAndTree> = ArrayList()
     private val id: String = branch.getId()
 
+    /**
+     * This records all changes as a single operation instead of a long list of fine-grained changes.
+     * It is assumed that the intended change is to put the model into the resulting state.
+     *
+     * @param subtreeRootNodeId if the update is not applied on the whole model, but only on a part of it,
+     *                          this ID specifies the root node of that subtree.
+     */
+    fun runBulkUpdate(subtreeRootNodeId: Long = ITree.ROOT_ID, body: () -> Unit) {
+        check(canWrite()) { "Only allowed inside a write transaction" }
+        if (bulkUpdateMode) return body()
+        try {
+            bulkUpdateMode = true
+            val baseTree = branch.transaction.tree as CLTree
+            body()
+            val resultTree = branch.transaction.tree as CLTree
+            currentOperations += BulkUpdateOp(KVEntryReference(resultTree.data), subtreeRootNodeId).afterApply(baseTree)
+        } finally {
+            bulkUpdateMode = false
+        }
+    }
+
+    fun isInBulkMode() = bulkUpdateMode
+
     fun operationApplied(op: IAppliedOperation) {
         check(canWrite()) { "Only allowed inside a write transaction" }
-        currentOperations.add(op)
+        if (!bulkUpdateMode) {
+            currentOperations.add(op)
+        }
     }
 
     override fun getId(): String {

model-datastructure/src/commonMain/kotlin/org/modelix/model/persistent/OperationSerializer.kt

@@ -24,6 +24,7 @@ import org.modelix.model.lazy.KVEntryReference
 import org.modelix.model.operations.AddNewChildOp
 import org.modelix.model.operations.AddNewChildSubtreeOp
 import org.modelix.model.operations.AddNewChildrenOp
+import org.modelix.model.operations.BulkUpdateOp
 import org.modelix.model.operations.DeleteNodeOp
 import org.modelix.model.operations.IOperation
 import org.modelix.model.operations.MoveNodeOp
@@ -130,6 +131,19 @@ class OperationSerializer private constructor() {
                     }
                 },
             )
+            INSTANCE.registerSerializer(
+                BulkUpdateOp::class,
+                object : Serializer<BulkUpdateOp> {
+                    override fun serialize(op: BulkUpdateOp): String {
+                        return longToHex(op.subtreeRootId) + SEPARATOR + op.resultTreeHash.getHash()
+                    }
+
+                    override fun deserialize(serialized: String): BulkUpdateOp {
+                        val parts = serialized.split(SEPARATOR).toTypedArray()
+                        return BulkUpdateOp(KVEntryReference(parts[1], CPTree.DESERIALIZER), longFromHex(parts[0]))
+                    }
+                },
+            )
             INSTANCE.registerSerializer(
                 DeleteNodeOp::class,
                 object : Serializer<DeleteNodeOp> {