Merge pull request #1515 from modelix/sync-plugin-using-original-ids

mps-sync-plugin: Preserve identity when importing MPS models into model-server

Author: slisson

.github/CODEOWNERS

@@ -12,6 +12,10 @@ interface IModelMask {
             .toSet()
         return children.filter { included.contains(it) }
     }
+
+    fun <T : IReadableNode> getFilteredChildren(parent: T): List<T> {
+        return filterChildren(parent, parent.getAllChildren() as List<T>)
+    }
 }
 
 class UnfilteredModelMask : IModelMask {

bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/IModelMask.kt

@@ -0,0 +1,34 @@
+package org.modelix.model.sync.bulk
+
+import org.modelix.model.api.IMutableModel
+import org.modelix.model.api.INodeReference
+import org.modelix.model.api.IReadableNode
+import org.modelix.model.api.IWritableNode
+
+/**
+ * A trivial implementation that expects the source and target nodes to have the same identity,
+ * meaning they can both be resolved by the same INodeReference.
+ */
+class IdentityPreservingNodeAssociation(
+    val targetModel: IMutableModel,
+    val overrides: Map<INodeReference, INodeReference>,
+) : INodeAssociation {
+
+    override fun resolveTarget(sourceNode: IReadableNode): IWritableNode? {
+        val sourceReference = sourceNode.getNodeReference()
+        val targetReference = overrides[sourceReference] ?: sourceReference
+        return targetModel.tryResolveNode(targetReference)
+    }
+
+    override fun associate(
+        sourceNode: IReadableNode,
+        targetNode: IWritableNode,
+    ) {
+        val sourceReference = sourceNode.getNodeReference()
+        val expectedTargetReference = overrides[sourceReference] ?: sourceReference
+        val actualTargetReference = targetNode.getNodeReference()
+        require(expectedTargetReference == actualTargetReference) {
+            "Cannot associate $sourceReference with $actualTargetReference, expected: $expectedTargetReference"
+        }
+    }
+}

bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/IdentityPreservingNodeAssociation.kt

@@ -119,7 +119,7 @@ class ModelSynchronizer(
                 syncChildren(sourceNode, conceptCorrectedTargetNode, forceSyncDescendants)
             }
         } else if (filter.needsDescentIntoSubtree(sourceNode)) {
-            for (sourceChild in sourceMask.filterChildren(sourceNode, sourceNode.getAllChildren())) {
+            for (sourceChild in sourceMask.getFilteredChildren(sourceNode)) {
                 runSafe {
                     val targetChild = nodeAssociation.resolveTarget(sourceChild)
                         ?: error("Expected target node was not found. sourceChild=${sourceChild.getNodeReference()}, originalId=${sourceChild.getOriginalReference()}")
@@ -173,8 +173,8 @@ class ModelSynchronizer(
 
     private fun syncChildren(sourceParent: IReadableNode, targetParent: IWritableNode, forceSyncDescendants: Boolean) {
         iterateMergedRoles(
-            sourceParent.getAllChildren().map { it.getContainmentLink() }.distinct(),
-            targetParent.getAllChildren().map { it.getContainmentLink() }.distinct(),
+            sourceMask.getFilteredChildren(sourceParent).map { it.getContainmentLink() }.distinct(),
+            targetMask.getFilteredChildren(targetParent).map { it.getContainmentLink() }.distinct(),
         ) { role ->
             runSafe {
                 syncChildrenInRole(sourceParent, role, targetParent, forceSyncDescendants)
@@ -215,7 +215,7 @@ class ModelSynchronizer(
             }
         }
 
-        val isOrdered = targetParent.isOrdered(role)
+        val isOrdered = targetParent.isOrdered(role) && sourceParent.isOrdered(role)
 
         // optimization for when there is no change in the child list
         if (associatedChildren.all { it.alreadyMatches(isOrdered) }) {

bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/ModelSynchronizer.kt

@@ -61,7 +61,7 @@ class NodeAssociationFromModelServer(val branch: IBranch, val targetModel: IMuta
 
     override fun resolveTarget(sourceNode: IReadableNode): IWritableNode? {
         return (pendingAssociations[nodeId(sourceNode)] ?: sourceNode.getOriginalReference())
-            ?.let { targetModel.resolveNode(NodeReference(it)) }
+            ?.let { targetModel.tryResolveNode(NodeReference(it)) }
     }
 
     override fun associate(sourceNode: IReadableNode, targetNode: IWritableNode) {

bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/NodeAssociationToModelServer.kt

@@ -30,7 +30,7 @@ class TransientNodeAssociation(val writeOriginalIds: Boolean, val targetModel: I
     override fun resolveTarget(sourceNode: IReadableNode): IWritableNode? {
         val ref = sourceNode.getOriginalOrCurrentReference()
         return associations[ref]
-            ?: targetModel.resolveNode(NodeReference(ref))
+            ?: targetModel.tryResolveNode(NodeReference(ref))
     }
 
     override fun associate(

bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/TransientNodeAssociation.kt

@@ -1,6 +1,7 @@
 package org.modelix.model.sync.bulk
 
 import org.modelix.model.api.IModel
+import org.modelix.model.api.INodeReference
 import org.modelix.model.api.IReadableNode
 import org.modelix.model.api.ITree
 import org.modelix.model.api.NodeReference
@@ -27,13 +28,13 @@ class InvalidationTree(sizeLimit: Int = 100_000) : GenericInvalidationTree<Long,
     }
 }
 
-class DefaultInvalidationTree(val root: NodeReference, sizeLimit: Int = 100_000) :
-    GenericInvalidationTree<NodeReference, IModel>(root, sizeLimit = sizeLimit) {
+class DefaultInvalidationTree(val root: INodeReference, sizeLimit: Int = 100_000) :
+    GenericInvalidationTree<INodeReference, IModel>(root, sizeLimit = sizeLimit) {
     override fun ancestorsAndSelf(
         model: IModel,
-        nodeId: NodeReference,
-    ): List<NodeReference> {
-        return model.resolveNode(nodeId)
+        nodeId: INodeReference,
+    ): List<INodeReference> {
+        return model.tryResolveNode(nodeId)
             ?.ancestors(includeSelf = true)
             ?.map { it.getNodeReference().toSerialized() }
             ?.toList()

bulk-model-sync-lib/src/jvmMain/kotlin/org/modelix/model/sync/bulk/InvalidationTree.kt

@@ -1,22 +1,22 @@
 package org.modelix.datastructures.btree
 
 import org.modelix.datastructures.objects.IObjectGraph
-import org.modelix.kotlin.utils.DelicateModelixApi
 import org.modelix.streams.IStream
+import org.modelix.streams.IStreamExecutorProvider
+import org.modelix.streams.getBlocking
 
-data class BTree<K, V>(val root: BTreeNode<K, V>) {
+data class BTree<K, V>(val root: BTreeNode<K, V>) : IStreamExecutorProvider by root.config.graph {
     constructor(config: BTreeConfig<K, V>) : this(BTreeNodeLeaf(config, emptyList()))
 
     val graph: IObjectGraph get() = root.config.graph
 
     fun validate() {
         graph.getStreamExecutor().query {
             root.validate(true)
-            @OptIn(DelicateModelixApi::class)
-            check(root.getEntries().toList().getSynchronous().map { it.key }.toSet().size == root.getEntries().map { it.key }.count().getSynchronous()) {
+            check(root.getEntries().toList().getBlocking(graph).map { it.key }.toSet().size == root.getEntries().map { it.key }.count().getBlocking(graph)) {
                 "duplicate entries: $root"
             }
-            check(root.getEntries().map { it.key }.toList().getSynchronous().sortedWith(root.config.keyConfiguration) == root.getEntries().map { it.key }.toList().getSynchronous()) {
+            check(root.getEntries().map { it.key }.toList().getBlocking(graph).sortedWith(root.config.keyConfiguration) == root.getEntries().map { it.key }.toList().getBlocking(graph)) {
                 "not sorted: $this"
             }
             IStream.of(Unit)

datastructures/src/commonMain/kotlin/org/modelix/datastructures/btree/BTree.kt

@@ -3,6 +3,8 @@ package org.modelix.datastructures.btree
 import org.modelix.datastructures.objects.IObjectData
 import org.modelix.datastructures.objects.IObjectDeserializer
 import org.modelix.datastructures.objects.IObjectReferenceFactory
+import org.modelix.datastructures.objects.Object
+import org.modelix.datastructures.objects.getDescendantsAndSelf
 import org.modelix.datastructures.serialization.SerializationSeparators
 import org.modelix.streams.IStream
 
@@ -33,6 +35,11 @@ sealed class BTreeNode<K, V> : IObjectData {
     fun splitIfNecessary(): Replacement<K, V> = if (isOverfilled()) split() else Replacement.Single(this)
     abstract fun mergeWithSibling(knownSeparator: K, right: BTreeNode<K, V>): BTreeNode<K, V>
 
+    override fun objectDiff(self: Object<*>, oldObject: Object<*>?): IStream.Many<Object<*>> {
+        // TODO performance
+        return self.getDescendantsAndSelf()
+    }
+
     class Deserializer<K, V>(val config: BTreeConfig<K, V>) : IObjectDeserializer<BTreeNode<K, V>> {
         override fun deserialize(
             serialized: String,

datastructures/src/commonMain/kotlin/org/modelix/datastructures/btree/BTreeNode.kt

@@ -0,0 +1,132 @@
+package org.modelix.datastructures.list
+
+import kotlinx.serialization.KSerializer
+import kotlinx.serialization.builtins.ListSerializer
+import kotlinx.serialization.builtins.serializer
+import kotlinx.serialization.descriptors.SerialDescriptor
+import kotlinx.serialization.encoding.Decoder
+import kotlinx.serialization.encoding.Encoder
+import org.modelix.datastructures.objects.IDataTypeConfiguration
+import org.modelix.datastructures.objects.IObjectData
+import org.modelix.datastructures.objects.IObjectDeserializer
+import org.modelix.datastructures.objects.IObjectGraph
+import org.modelix.datastructures.objects.IObjectReferenceFactory
+import org.modelix.datastructures.objects.Object
+import org.modelix.datastructures.objects.ObjectHash
+import org.modelix.datastructures.objects.ObjectReference
+import org.modelix.datastructures.objects.getDescendantsAndSelf
+import org.modelix.datastructures.objects.getHashString
+import org.modelix.datastructures.serialization.SerializationSeparators
+import org.modelix.kotlin.utils.urlDecode
+import org.modelix.kotlin.utils.urlEncode
+import org.modelix.streams.IStream
+import kotlin.collections.chunked
+
+class LargeListConfig<E>(
+    val graph: IObjectGraph,
+    val elementType: IDataTypeConfiguration<E>,
+    val maxNodeSize: Int = 20,
+) : IObjectDeserializer<LargeList<E>> {
+    override fun deserialize(input: String, referenceFactory: IObjectReferenceFactory): LargeList<E> {
+        val data = if (input.startsWith(LargeList.LARGE_LIST_PREFIX)) {
+            val subLists = input.substring(LargeList.LARGE_LIST_PREFIX.length)
+                .split(SerializationSeparators.LEVEL2)
+                .map { referenceFactory(it, this) }
+            LargeListInternalNode(this, subLists)
+        } else {
+            LargeListLeafNode(
+                this,
+                input.split(SerializationSeparators.LEVEL2)
+                    .filter { it.isNotEmpty() }
+                    .map { elementType.deserialize(it.urlDecode()!!) },
+            )
+        }
+        return data
+    }
+
+    fun createEmptyList(): LargeList<E> = LargeListLeafNode(this, emptyList())
+
+    fun createList(elements: List<E>): LargeList<E> {
+        return if (elements.size <= maxNodeSize) {
+            LargeListLeafNode(this, elements)
+        } else {
+            // split the elements into at most maxNodeSize sub lists, but also minimize the number of objects
+            val sublistSizes = ((elements.size + maxNodeSize - 1) / maxNodeSize).coerceAtLeast(maxNodeSize)
+            LargeListInternalNode(this, elements.chunked(sublistSizes) { graph.fromCreated(createList(it.toList())) }.toList())
+        }
+    }
+}
+
+class LargeListKSerializer<E>(val config: LargeListConfig<E>) : KSerializer<LargeList<E>> {
+    private val listSerializer = ListSerializer(String.serializer())
+    override val descriptor: SerialDescriptor = listSerializer.descriptor
+
+    override fun serialize(encoder: Encoder, value: LargeList<E>) {
+        when (value) {
+            is LargeListInternalNode<E> -> listSerializer.serialize(encoder, value.subLists.map { it.getHashString() })
+            is LargeListLeafNode<E> -> listSerializer.serialize(encoder, value.elements.map { config.elementType.serialize(it) })
+        }
+    }
+
+    override fun deserialize(decoder: Decoder): LargeList<E> {
+        val strings = listSerializer.deserialize(decoder)
+        return if (strings.isNotEmpty() && ObjectHash.isValidHashString(strings.first())) {
+            LargeListInternalNode(
+                config,
+                strings.map { config.graph.fromHashString(it, config) },
+            )
+        } else {
+            LargeListLeafNode(config, strings.map { config.elementType.deserialize(it) })
+        }
+    }
+}
+
+sealed class LargeList<E>() : IObjectData {
+    companion object {
+        const val LARGE_LIST_PREFIX = "OL" + SerializationSeparators.LEVEL1
+    }
+
+    abstract fun getElements(): IStream.Many<E>
+
+    override fun objectDiff(self: Object<*>, oldObject: Object<*>?): IStream.Many<Object<*>> {
+        return self.getDescendantsAndSelf()
+    }
+}
+
+class LargeListInternalNode<E>(val config: LargeListConfig<E>, val subLists: List<ObjectReference<LargeList<E>>>) : LargeList<E>() {
+    override fun serialize(): String {
+        return LARGE_LIST_PREFIX + subLists.joinToString(SerializationSeparators.LEVEL2) { it.getHashString() }
+    }
+
+    override fun getDeserializer(): IObjectDeserializer<LargeList<E>> = config
+
+    override fun getContainmentReferences(): List<ObjectReference<IObjectData>> {
+        return subLists.toList()
+    }
+
+    override fun getElements(): IStream.Many<E> {
+        return IStream.many(subLists).flatMap {
+            it.resolveData().flatMap { it.getElements() }
+        }
+    }
+}
+
+class LargeListLeafNode<E>(val config: LargeListConfig<E>, val elements: List<E>) : LargeList<E>() {
+    override fun serialize(): String {
+        return if (elements.isEmpty()) {
+            ""
+        } else {
+            elements.joinToString(SerializationSeparators.LEVEL2) { config.elementType.serialize(it).urlEncode() }
+        }
+    }
+
+    override fun getDeserializer(): IObjectDeserializer<LargeList<E>> = config
+
+    override fun getContainmentReferences(): List<ObjectReference<IObjectData>> {
+        return elements.flatMap { config.elementType.getContainmentReferences(it) }
+    }
+
+    override fun getElements(): IStream.Many<E> {
+        return IStream.many(elements)
+    }
+}