feat(mps-sync-plugin): import MPS nodes with their original ID

Author: slisson

bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/IdentityPreservingNodeAssociation.kt

@@ -0,0 +1,34 @@
+package org.modelix.model.sync.bulk
+
+import org.modelix.model.api.IMutableModel
+import org.modelix.model.api.INodeReference
+import org.modelix.model.api.IReadableNode
+import org.modelix.model.api.IWritableNode
+
+/**
+ * A trivial implementation that expects the source and target nodes to have the same identity,
+ * meaning they can both be resolved by the same INodeReference.
+ */
+class IdentityPreservingNodeAssociation(
+    val targetModel: IMutableModel,
+    val overrides: Map<INodeReference, INodeReference>,
+) : INodeAssociation {
+
+    override fun resolveTarget(sourceNode: IReadableNode): IWritableNode? {
+        val sourceReference = sourceNode.getNodeReference()
+        val targetReference = overrides[sourceReference] ?: sourceReference
+        return targetModel.resolveNode(targetReference)
+    }
+
+    override fun associate(
+        sourceNode: IReadableNode,
+        targetNode: IWritableNode,
+    ) {
+        val sourceReference = sourceNode.getNodeReference()
+        val expectedTargetReference = overrides[sourceReference] ?: sourceReference
+        val actualTargetReference = targetNode.getNodeReference()
+        require(expectedTargetReference == actualTargetReference) {
+            "Cannot associate $sourceReference with $actualTargetReference, expected: $expectedTargetReference"
+        }
+    }
+}

bulk-model-sync-lib/src/jvmMain/kotlin/org/modelix/model/sync/bulk/InvalidationTree.kt

@@ -1,6 +1,7 @@
 package org.modelix.model.sync.bulk
 
 import org.modelix.model.api.IModel
+import org.modelix.model.api.INodeReference
 import org.modelix.model.api.IReadableNode
 import org.modelix.model.api.ITree
 import org.modelix.model.api.NodeReference
@@ -27,12 +28,12 @@ class InvalidationTree(sizeLimit: Int = 100_000) : GenericInvalidationTree<Long,
     }
 }
 
-class DefaultInvalidationTree(val root: NodeReference, sizeLimit: Int = 100_000) :
-    GenericInvalidationTree<NodeReference, IModel>(root, sizeLimit = sizeLimit) {
+class DefaultInvalidationTree(val root: INodeReference, sizeLimit: Int = 100_000) :
+    GenericInvalidationTree<INodeReference, IModel>(root, sizeLimit = sizeLimit) {
     override fun ancestorsAndSelf(
         model: IModel,
-        nodeId: NodeReference,
-    ): List<NodeReference> {
+        nodeId: INodeReference,
+    ): List<INodeReference> {
         return model.resolveNode(nodeId)
             ?.ancestors(includeSelf = true)
             ?.map { it.getNodeReference().toSerialized() }

datastructures/src/commonMain/kotlin/org/modelix/datastructures/patricia/PatriciaNode.kt

@@ -1,7 +1,11 @@
 package org.modelix.datastructures.patricia
 
+import org.modelix.datastructures.EntryAddedEvent
+import org.modelix.datastructures.EntryChangedEvent
+import org.modelix.datastructures.EntryRemovedEvent
 import org.modelix.datastructures.IPersistentMap
 import org.modelix.datastructures.IPersistentMapRootData
+import org.modelix.datastructures.MapChangeEvent
 import org.modelix.datastructures.objects.IObjectData
 import org.modelix.datastructures.objects.IObjectDeserializer
 import org.modelix.datastructures.objects.IObjectGraph
@@ -14,11 +18,18 @@ import org.modelix.datastructures.serialization.SplitJoinSerializer
 import org.modelix.kotlin.utils.urlDecode
 import org.modelix.kotlin.utils.urlEncode
 import org.modelix.streams.IStream
+import org.modelix.streams.flatten
+import org.modelix.streams.ifEmpty
 import org.modelix.streams.plus
 
 data class PatriciaNode<K, V : Any>(
     val config: PatriciaTrieConfig<K, V>,
     val ownPrefix: String,
+
+    /**
+     * Equivalent to `children.map { it.ownPrefix.first() }.join("")`, just allows choosing the correct child without
+     * resolving all of them. Sorted.
+     */
     val firstChars: String,
     val children: List<ObjectReference<PatriciaNode<K, V>>>,
 
@@ -72,6 +83,10 @@ data class PatriciaNode<K, V : Any>(
 
     fun withValue(newValue: V?) = copy(value = newValue)
 
+    /**
+     * Returns all entries.
+     * @param prefix the prefix from the root to this node. Required to assemble the key.
+     */
     fun getEntries(prefix: CharSequence): IStream.Many<Pair<CharSequence, V>> {
         val fullPrefix = prefix + ownPrefix
         val descendants = IStream.Companion.many(children).flatMap { it.resolveData() }
@@ -135,40 +150,61 @@ data class PatriciaNode<K, V : Any>(
         }
     }
 
-    fun put(newKey: CharSequence, newValue: V?): IStream.ZeroOrOne<PatriciaNode<K, V>> {
+    fun updateSubtree(newKey: CharSequence, updater: (PatriciaNode<K, V>) -> IStream.ZeroOrOne<PatriciaNode<K, V>>): IStream.ZeroOrOne<PatriciaNode<K, V>> {
         val commonPrefix = newKey.commonPrefixWith(this.ownPrefix)
         val remainingNewKey = newKey.drop(commonPrefix.length)
         val remainingOwnPrefix = this.ownPrefix.drop(commonPrefix.length)
         return (
             if (remainingOwnPrefix.isEmpty() && remainingNewKey.isEmpty()) {
-                // key references this node -> overwrite the value
-                IStream.of(withValue(newValue))
+                // key references this node
+                updater(this)
             } else if (remainingOwnPrefix.isEmpty()) {
                 // key is longer -> insert into children
                 val index = firstChars.binarySearch(remainingNewKey.first())
                 if (index >= 0) {
                     children[index].resolveData()
-                        .flatMapOne { it.put(remainingNewKey, newValue).orNull() }
+                        .flatMapOne { it.updateSubtree(remainingNewKey, updater).orNull() }
                         .mapNotNull { withChildReplacedNullable(index, it) }
                 } else {
                     val insertionIndex = (-index) - 1
-                    val newChild = PatriciaNode<K, V>(
+                    PatriciaNode<K, V>(
                         config = config,
                         ownPrefix = remainingNewKey.toString(),
-                        value = newValue,
+                        value = null,
                         firstChars = "",
                         children = emptyList(),
-                    )
-                    IStream.of(withChildInserted(insertionIndex, remainingNewKey.first(), newChild))
+                    ).let { updater(it) }
+                        .map { withChildInserted(insertionIndex, remainingNewKey.first(), it) }
+                        .ifEmpty {
+                            // updater returned an empty node, and since this part is about inserting, nothing changed
+                            this
+                        }
                 }
             } else {
                 // key is shorter -> need to split into a node with a shorter prefix
-                split(commonPrefix).put(newKey, newValue)
+                split(commonPrefix).updateSubtree(newKey, updater)
             }
             ).flatMapZeroOrOne { it.tryMerge() }
     }
 
-    fun split(commonPrefix: CharSequence): PatriciaNode<K, V> {
+    fun put(newKey: CharSequence, newValue: V?): IStream.ZeroOrOne<PatriciaNode<K, V>> {
+        return updateSubtree(newKey) {
+            IStream.of(it.copy(value = newValue))
+        }
+    }
+
+    fun replaceSubtree(prefix: CharSequence, newSubtree: PatriciaNode<K, V>?): IStream.ZeroOrOne<PatriciaNode<K, V>> {
+        return updateSubtree(prefix) {
+            IStream.ofNotNull(newSubtree?.copy(ownPrefix = it.ownPrefix))
+        }
+    }
+
+    /**
+     * Shortens the prefix of this node to the given one as a preparation for inserting children or setting a value.
+     */
+    private fun split(commonPrefix: CharSequence): PatriciaNode<K, V> {
+        require(ownPrefix.startsWith(commonPrefix))
+        if (ownPrefix.length == commonPrefix.length) return this
         val remainingPrefix = this.ownPrefix.drop(commonPrefix.length)
         return PatriciaNode<K, V>(
             config = config,
@@ -195,7 +231,7 @@ data class PatriciaNode<K, V : Any>(
         return ownPrefix.urlEncode() +
             S1 + firstChars.urlEncode() +
             S1 + children.joinToString(S2.toString()) { it.getHashString() } +
-            S1 + value
+            S1 + value?.let { config.valueConfig.serialize(it) }.urlEncode()
     }
 
     override fun getDeserializer(): IObjectDeserializer<*> {
@@ -206,6 +242,79 @@ data class PatriciaNode<K, V : Any>(
         return children + (value?.let { config.valueConfig.getContainmentReferences(it) } ?: emptyList())
     }
 
+    fun getChanges(path: CharSequence, oldNode: PatriciaNode<K, V>?, changesOnly: Boolean): IStream.Many<MapChangeEvent<K, V>> {
+        if (oldNode == null) {
+            return if (changesOnly) {
+                IStream.empty()
+            } else {
+                getEntries(path).map { EntryAddedEvent(config.keyConfig.deserialize(it.first.toString()), it.second) }
+            }
+        }
+        return if (ownPrefix == oldNode.ownPrefix) {
+            val pathForChildren = path + ownPrefix
+            val matchingChildren = if (firstChars == oldNode.firstChars) {
+                children.zip(oldNode.children)
+            } else {
+                val newChildren = firstChars.asSequence().zip(children.asSequence()).toMap()
+                val oldChildren = oldNode.firstChars.asSequence().zip(oldNode.children.asSequence()).toMap()
+                val allFirstChars = newChildren.keys.plus(oldChildren.keys).distinct()
+                allFirstChars.map { newChildren[it] to oldChildren[it] }
+            }
+            val changesFromChildren = IStream.many(matchingChildren).flatMap { (newChildRef, oldChildRef) ->
+                val newChild = newChildRef?.resolveData() ?: IStream.of(null)
+                val oldChild = oldChildRef?.resolveData() ?: IStream.of(null)
+
+                newChild.zipWith(oldChild) { newChild, oldChild ->
+                    if (newChild == null) {
+                        if (oldChild == null) {
+                            IStream.empty()
+                        } else {
+                            if (changesOnly) {
+                                IStream.empty()
+                            } else {
+                                oldChild.getEntries(pathForChildren).map {
+                                    EntryRemovedEvent(config.keyConfig.deserialize(it.first.toString()), it.second)
+                                }
+                            }
+                        }
+                    } else {
+                        newChild.getChanges(pathForChildren, oldChild, changesOnly)
+                    }
+                }
+            }.flatten()
+
+            fun ownKey() = config.keyConfig.deserialize(pathForChildren.toString())
+            val ownChange = if (this.value == null) {
+                if (oldNode.value == null) {
+                    IStream.empty()
+                } else {
+                    IStream.of(EntryRemovedEvent(ownKey(), oldNode.value))
+                }
+            } else {
+                if (oldNode.value == null) {
+                    IStream.of(EntryAddedEvent(ownKey(), this.value))
+                } else {
+                    if (config.equal(this.value, oldNode.value)) {
+                        IStream.empty()
+                    } else {
+                        IStream.of(
+                            EntryChangedEvent(
+                                key = config.keyConfig.deserialize(pathForChildren.toString()),
+                                oldValue = oldNode.value,
+                                newValue = this.value,
+                            ),
+                        )
+                    }
+                }
+            }
+
+            ownChange + changesFromChildren
+        } else {
+            val commonPrefix = ownPrefix.commonPrefixWith(oldNode.ownPrefix)
+            split(commonPrefix).getChanges(path, oldNode.split(commonPrefix), changesOnly)
+        }
+    }
+
     class Deserializer<K, V : Any>(val config: (IObjectGraph) -> PatriciaTrieConfig<K, V>) : IObjectDeserializer<PatriciaNode<K, V>> {
         constructor(config: PatriciaTrieConfig<K, V>) : this({ config })
         override fun deserialize(
@@ -221,7 +330,7 @@ data class PatriciaNode<K, V : Any>(
                 parts[0].urlDecode()!!,
                 parts[1].urlDecode()!!,
                 parts[2].split(S2).filter { it.isNotEmpty() }.map { referenceFactory.fromHashString(it, this) },
-                config.valueConfig.deserialize(parts[3]),
+                parts[3].urlDecode()?.let { config.valueConfig.deserialize(it) },
             )
         }
     }

datastructures/src/commonMain/kotlin/org/modelix/datastructures/patricia/PatriciaTrie.kt

@@ -36,21 +36,40 @@ class PatriciaTrie<K, V : Any>(
         return PatriciaTrie(config, (newRoot ?: PatriciaNode(config)).asObject(config.graph))
     }
 
+    private fun IStream.ZeroOrOne<PatriciaNode<K, V>>.withNewRoot() = orNull().map { withNewRoot(it) }
+
     override fun get(key: K): IStream.ZeroOrOne<V> {
         return root.data.get(keyAsString(key))
     }
 
     override fun put(key: K, value: V): IStream.One<PatriciaTrie<K, V>> {
         val keyString = keyAsString(key)
-        return root.data.put(keyString, value).orNull().map { withNewRoot(it) }
+        return root.data.put(keyString, value).withNewRoot()
     }
 
+    /**
+     * Returns a copy of the tree that contains only those entries starting with the given prefix.
+     */
     fun slice(prefix: CharSequence): IStream.One<PatriciaTrie<K, V>> {
-        return root.data.slice(prefix).orNull().map { withNewRoot(it) }
+        return root.data.slice(prefix).withNewRoot()
+    }
+
+    /**
+     * Removes all entries starting with the given [prefix] and adds all entries from [newEntries] starting with the
+     * same prefix.
+     * This allows efficiently updating a whole subtree from an external source without having to compare the new and
+     * existing data. It can just be reimported and then compared by using the builtin diff support of this
+     * data structure.
+     */
+    fun replaceSlice(prefix: CharSequence, newEntries: PatriciaTrie<K, V>): IStream.One<PatriciaTrie<K, V>> {
+        if (prefix.isEmpty()) return IStream.of(newEntries)
+        return newEntries.root.data.getSubtree(prefix).orNull().flatMapOne { replacement ->
+            root.data.replaceSubtree(prefix, replacement).withNewRoot()
+        }
     }
 
     override fun remove(key: K): IStream.One<PatriciaTrie<K, V>> {
-        return root.data.put(keyAsString(key), null).orNull().map { withNewRoot(it) }
+        return root.data.put(keyAsString(key), null).withNewRoot()
     }
 
     override fun getAllValues(): IStream.Many<V> {
@@ -66,7 +85,8 @@ class PatriciaTrie<K, V : Any>(
     }
 
     override fun getAll(keys: Iterable<K>): IStream.Many<Pair<K, V>> {
-        TODO("Not yet implemented")
+        // TODO performance
+        return IStream.many(keys).flatMap { key -> get(key).map { key to it } }
     }
 
     override fun putAll(entries: Iterable<Pair<K, V>>): IStream.One<IPersistentMap<K, V>> {
@@ -75,7 +95,8 @@ class PatriciaTrie<K, V : Any>(
     }
 
     override fun removeAll(keys: Iterable<K>): IStream.One<IPersistentMap<K, V>> {
-        TODO("Not yet implemented")
+        // TODO performance
+        return keys.fold(IStream.of(this)) { acc, key -> acc.flatMapOne { it.remove(key) } }
     }
 
     override fun removeAllEntries(entries: Iterable<Pair<K, V>>): IStream.One<IPersistentMap<K, V>> {
@@ -86,7 +107,8 @@ class PatriciaTrie<K, V : Any>(
         oldMap: IPersistentMap<K, V>,
         changesOnly: Boolean,
     ): IStream.Many<MapChangeEvent<K, V>> {
-        TODO("Not yet implemented")
+        oldMap as PatriciaTrie<K, V>
+        return root.data.getChanges("", oldMap.root.data, changesOnly)
     }
 
     override fun equals(other: Any?): Boolean {

datastructures/src/commonMain/kotlin/org/modelix/datastructures/patricia/PatriciaTrieConfig.kt

@@ -2,9 +2,29 @@ package org.modelix.datastructures.patricia
 
 import org.modelix.datastructures.objects.IDataTypeConfiguration
 import org.modelix.datastructures.objects.IObjectGraph
+import kotlin.jvm.JvmName
 
 class PatriciaTrieConfig<K, V>(
     val graph: IObjectGraph,
     val keyConfig: IDataTypeConfiguration<K>,
     val valueConfig: IDataTypeConfiguration<V>,
-)
+) {
+
+    @JvmName("keysEqual")
+    fun equal(a: K, b: K): Boolean {
+        return if (a == null) {
+            b == null
+        } else {
+            b != null && keyConfig.equal(a, b)
+        }
+    }
+
+    @JvmName("valuesEqual")
+    fun equal(a: V?, b: V?): Boolean {
+        return if (a == null) {
+            b == null
+        } else {
+            b != null && valueConfig.equal(a, b)
+        }
+    }
+}