Merge pull request #1520 from modelix/git-import

Incremental Git import

Author: slisson

bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/ModelSynchronizer.kt

@@ -8,7 +8,6 @@ import org.modelix.model.api.IReferenceLinkReference
 import org.modelix.model.api.IRoleReference
 import org.modelix.model.api.IWritableNode
 import org.modelix.model.api.NewNodeSpec
-import org.modelix.model.api.getOriginalReference
 import org.modelix.model.api.isOrdered
 import org.modelix.model.api.matches
 import org.modelix.model.api.mergeWith
@@ -100,37 +99,49 @@ class ModelSynchronizer(
     ): Unit = withPushSyncStack(sourceNode) {
         nodeAssociation.associate(sourceNode, targetNode)
         if (forceSyncDescendants || filter.needsSynchronization(sourceNode)) {
-            LOG.trace { "Synchronizing changed node. sourceNode = $sourceNode" }
-            runSafe { synchronizeProperties(sourceNode, targetNode) }
-            runSafe { synchronizeReferences(sourceNode, targetNode) }
-
-            val conceptCorrectedTargetNode = runSafe {
-                val sourceConcept = sourceNode.getConceptReference()
-                val targetConcept = targetNode.getConceptReference()
-
-                if (sourceConcept != targetConcept) {
-                    targetNode.changeConcept(sourceConcept)
-                } else {
-                    targetNode
-                }
-            }.getOrDefault(targetNode)
-
-            runSafe {
-                syncChildren(sourceNode, conceptCorrectedTargetNode, forceSyncDescendants)
-            }
+            synchronizeNodeAndChildren(sourceNode, targetNode, forceSyncDescendants)
         } else if (filter.needsDescentIntoSubtree(sourceNode)) {
-            for (sourceChild in sourceMask.getFilteredChildren(sourceNode)) {
-                runSafe {
-                    val targetChild = nodeAssociation.resolveTarget(sourceChild)
-                        ?: error("Expected target node was not found. sourceChild=${sourceChild.getNodeReference()}, originalId=${sourceChild.getOriginalReference()}")
-                    synchronizeNode(sourceChild, targetChild, forceSyncDescendants)
+            val matchingChildren = sourceMask.getFilteredChildren(sourceNode).map { it to nodeAssociation.resolveTarget(it) }
+            if (matchingChildren.any { it.second == null }) {
+                // Inconsistent information provided by filter. Apparently, the node itself needs synchronization.
+                synchronizeNodeAndChildren(sourceNode, targetNode, forceSyncDescendants)
+            } else {
+                for ((sourceChild, targetChild) in matchingChildren) {
+                    runSafe {
+                        synchronizeNode(sourceChild, targetChild!!, forceSyncDescendants)
+                    }
                 }
             }
         } else {
             LOG.trace { "Skipping subtree due to filter. root = $sourceNode" }
         }
     }
 
+    private fun synchronizeNodeAndChildren(
+        sourceNode: IReadableNode,
+        targetNode: IWritableNode,
+        forceSyncDescendants: Boolean,
+    ) {
+        LOG.trace { "Synchronizing changed node. sourceNode = $sourceNode" }
+        runSafe { synchronizeProperties(sourceNode, targetNode) }
+        runSafe { synchronizeReferences(sourceNode, targetNode) }
+
+        val conceptCorrectedTargetNode = runSafe {
+            val sourceConcept = sourceNode.getConceptReference()
+            val targetConcept = targetNode.getConceptReference()
+
+            if (sourceConcept != targetConcept) {
+                targetNode.changeConcept(sourceConcept)
+            } else {
+                targetNode
+            }
+        }.getOrDefault(targetNode)
+
+        runSafe {
+            syncChildren(sourceNode, conceptCorrectedTargetNode, forceSyncDescendants)
+        }
+    }
+
     private fun synchronizeReferences(
         sourceNode: IReadableNode,
         targetNode: IWritableNode,

bulk-model-sync-lib/src/jvmMain/kotlin/org/modelix/model/sync/bulk/InvalidationTree.kt

@@ -59,7 +59,7 @@ abstract class GenericInvalidationTree<ID, M>(root: ID, val sizeLimit: Int = 100
      */
     fun invalidate(containmentPath: List<ID>, includingDescendants: Boolean = false) {
         require(containmentPath[0] == rootNode.id) { "Path must start with the root node. Expected: ${rootNode.id}, was: $containmentPath" }
-        rootNode.invalidate(containmentPath, 0, includingDescendants)
+        rootNode.invalidate(containmentPath, 1, includingDescendants)
         rootNode.rebalance(sizeLimit)
     }
 
@@ -80,13 +80,25 @@ abstract class GenericInvalidationTree<ID, M>(root: ID, val sizeLimit: Int = 100
     fun hasAnyInvalidations(): Boolean = rootNode.hasAnyInvalidations()
 
     override fun needsDescentIntoSubtree(subtreeRoot: IReadableNode): Boolean {
-        return rootNode.needsDescentIntoSubtree(getContainmentPath(subtreeRoot), 0)
+        return needsDescentIntoSubtree(getContainmentPath(subtreeRoot))
+    }
+
+    fun needsDescentIntoSubtree(containmentPath: List<ID>): Boolean {
+        require(containmentPath[0] == rootNode.id) { "Path must start with the root node. Expected: ${rootNode.id}, was: $containmentPath" }
+        return rootNode.needsDescentIntoSubtree(containmentPath, 1)
     }
 
     override fun needsSynchronization(node: IReadableNode): Boolean {
-        return rootNode.nodeNeedsUpdate(getContainmentPath(node), 0)
+        return needsSynchronization(getContainmentPath(node))
+    }
+
+    fun needsSynchronization(containmentPath: List<ID>): Boolean {
+        require(containmentPath[0] == rootNode.id) { "Path must start with the root node. Expected: ${rootNode.id}, was: $containmentPath" }
+        return rootNode.nodeNeedsUpdate(containmentPath, 1)
     }
 
+    fun size() = rootNode.size()
+
     private class Node<E>(val id: E) {
         private var subtreeSize = 1
         private var nodeNeedsUpdate: Boolean = false
@@ -95,6 +107,8 @@ abstract class GenericInvalidationTree<ID, M>(root: ID, val sizeLimit: Int = 100
 
         fun hasAnyInvalidations() = nodeNeedsUpdate || allDescendantsNeedUpdate || invalidChildren.isNotEmpty()
 
+        fun size() = subtreeSize
+
         fun reset() {
             subtreeSize = 1
             nodeNeedsUpdate = false

commitlint.config.js

@@ -19,6 +19,7 @@ module.exports = {
         "model-server",
         "model-server-openapi",
         "modelql",
+        "mps-git-import-plugin",
         "mps-model-adapters",
         "mps-model-server",
         "mps-sync-plugin",

datastructures/src/commonMain/kotlin/org/modelix/datastructures/IPersistentMap.kt

@@ -3,6 +3,7 @@ package org.modelix.datastructures
 import org.modelix.datastructures.objects.IDataTypeConfiguration
 import org.modelix.datastructures.objects.IObjectData
 import org.modelix.datastructures.objects.Object
+import org.modelix.datastructures.objects.ObjectReference
 import org.modelix.streams.IStream
 import org.modelix.streams.IStreamExecutorProvider
 
@@ -37,3 +38,7 @@ interface IPersistentMapRootData<K, V> : IObjectData {
 fun <K, V> Object<IPersistentMapRootData<K, V>>.createMapInstance(): IPersistentMap<K, V> {
     return data.createMapInstance(this)
 }
+
+fun <K, V> ObjectReference<IPersistentMapRootData<K, V>>.createMapInstance(): IStream.One<IPersistentMap<K, V>> {
+    return resolve().map { it.createMapInstance() }
+}

datastructures/src/commonMain/kotlin/org/modelix/datastructures/objects/Object.kt

@@ -35,6 +35,10 @@ open class Object<out E : IObjectData>(val data: E, val ref: ObjectReference<E>)
     override fun hashCode(): Int {
         error("Use .getHash() for comparing objects")
     }
+
+    override fun toString(): String {
+        return getHashString() + " -> " + data.serialize()
+    }
 }
 
 /**

datastructures/src/commonMain/kotlin/org/modelix/datastructures/patricia/PatriciaNode.kt

@@ -376,13 +376,13 @@ data class PatriciaNode<K, V : Any>(
             IStream.of(self) + changesFromValue + changesFromChildren
         } else {
             val commonPrefix = ownPrefix.commonPrefixWith(oldNode.ownPrefix)
-            (
-                IStream.of(self) + self.split(commonPrefix).let {
-                    it.data.objectDiff(it, oldObject.split(commonPrefix), path).filter { it.graph == config.graph }
+            IStream.of(self) + (
+                self.split(commonPrefix).let {
+                    it.data.objectDiff(it, oldObject.split(commonPrefix), path)
                 }
                 ).filter {
                 // filter out the split result, which isn't part of the real object graph
-                it.graph == config.graph
+                it.graph == config.graph && it.getHash() != self.getHash()
             }
         }
     }
@@ -416,7 +416,7 @@ data class PatriciaNode<K, V : Any>(
 
 private class CharSequenceConcatenation(val a: CharSequence, val b: CharSequence) : CharSequence {
     override fun get(index: Int): Char {
-        return if (index < a.length) a.get(index) else b.get(index + a.length)
+        return if (index < a.length) a.get(index) else b.get(index - a.length)
     }
 
     override fun subSequence(startIndex: Int, endIndex: Int): CharSequence {

datastructures/src/commonMain/kotlin/org/modelix/datastructures/patricia/PatriciaTrie.kt

@@ -5,11 +5,14 @@ import org.modelix.datastructures.MapChangeEvent
 import org.modelix.datastructures.objects.IDataTypeConfiguration
 import org.modelix.datastructures.objects.IObjectGraph
 import org.modelix.datastructures.objects.Object
+import org.modelix.datastructures.objects.ObjectReference
 import org.modelix.datastructures.objects.StringDataTypeConfiguration
 import org.modelix.datastructures.objects.asObject
 import org.modelix.streams.IStream
 import org.modelix.streams.IStreamExecutorProvider
 
+typealias PatriciaTrieRef<K, V> = ObjectReference<PatriciaNode<K, V>>
+
 /**
  * Entries with the same prefix are put into the same subtree. The prefix has a variable length.
  */

datastructures/src/commonTest/kotlin/org/modelix/datastructures/TreeDiffTest.kt

@@ -66,4 +66,49 @@ class TreeDiffTest {
 
         assertTree()
     }
+
+    @Test
+    fun `diff returns no duplicate objects`() {
+        val graph = FullyLoadedObjectGraph()
+        var tree = PatriciaTrie.withStrings(graph)
+        var previousTrees: MutableList<PatriciaTrie<String, String>> = mutableListOf()
+
+        val alphabet = "abcd"
+        val rand = Random(5465)
+        fun randomString(length: Int) = (0 until length).joinToString("") { alphabet.random(rand).toString() }
+
+        val removedEntries = (1..1000).map { randomString(rand.nextInt(0, 6)) }.toMutableSet()
+        val values = removedEntries.associateWith { "value_of_$it" }
+        val addedEntries = mutableSetOf<String>()
+
+        fun assertTree() {
+            for (previousTree in (1..5).map { previousTrees.random(rand) }) {
+                val diff = tree.asObject().objectDiff(previousTree.asObject())
+                    .map { it.getHash() to it.data.serialize() }
+                    .toList().getBlocking(tree)
+
+                val duplicateObjects = diff.groupBy { it.first }.filter { it.value.size > 1 }.map { it.value.first() }
+                assertEquals(emptyList(), duplicateObjects)
+            }
+        }
+
+        repeat(1_000) {
+            previousTrees.add(tree)
+            if (removedEntries.size > addedEntries.size) {
+                val key = removedEntries.random(rand)
+                removedEntries.remove(key)
+                addedEntries.add(key)
+                tree = tree.put(key, values[key]!!).getBlocking(tree)
+                assertTree()
+            } else {
+                val key = addedEntries.random(rand)
+                removedEntries.add(key)
+                addedEntries.remove(key)
+                tree = tree.remove(key).getBlocking(tree)
+                assertTree()
+            }
+        }
+
+        assertTree()
+    }
 }

gradle/libs.versions.toml

@@ -42,6 +42,7 @@ kotest = "5.9.1"
 reaktive = "2.2.0"
 testcontainers = "1.20.6"
 keycloak = "26.0.4"
+arrow = "2.0.1"
 
 [libraries]
 
@@ -152,3 +153,20 @@ nimbus-jose-jwt = { group = "com.nimbusds", name = "nimbus-jose-jwt", version =
 bouncycastle-bcpkix = { group = "org.bouncycastle", name = "bcpkix-lts8on", version = "2.73.7" }
 
 kotlincrypto-sha2 = { module = "org.kotlincrypto.hash:sha2", version = "0.7.0" }
+
+arrow-core = { group = "io.arrow-kt", name = "arrow-core", version.ref = "arrow" }
+arrow-fx-coroutines = { group = "io.arrow-kt", name = "arrow-fx-coroutines", version.ref = "arrow" }
+arrow-arrow-stack = { group = "io.arrow-kt", name = "arrow-stack", version.ref = "arrow" }
+arrow-optics = { group = "io.arrow-kt", name = "arrow-optics", version.ref = "arrow" }
+arrow-optics-ksp-plugin = { group = "io.arrow-kt", name = "arrow-optics-ksp-plugin", version.ref = "arrow" }
+arrow-autoclose = { group = "io.arrow-kt", name = "arrow-autoclose", version.ref = "arrow" }
+arrow-resilience = { group = "io.arrow-kt", name = "arrow-resilience", version.ref = "arrow" }
+arrow-fx-stm = { group = "io.arrow-kt", name = "arrow-fx-stm", version.ref = "arrow" }
+arrow-atomic = { group = "io.arrow-kt", name = "arrow-atomic", version.ref = "arrow" }
+arrow-collectors = { group = "io.arrow-kt", name = "arrow-collectors", version.ref = "arrow" }
+arrow-eval = { group = "io.arrow-kt", name = "arrow-eval", version.ref = "arrow" }
+arrow-functions = { group = "io.arrow-kt", name = "arrow-functions", version.ref = "arrow" }
+arrow-core-high-arity = { group = "io.arrow-kt", name = "arrow-core-high-arity", version.ref = "arrow" }
+arrow-core-serialization = { group = "io.arrow-kt", name = "arrow-core-serialization", version.ref = "arrow" }
+arrow-cache4k = { group = "io.arrow-kt", name = "arrow-cache4k", version.ref = "arrow" }
+arrow-optics-compose = { group = "io.arrow-kt", name = "arrow-optics-compose", version.ref = "arrow" }

model-api/src/commonMain/kotlin/org/modelix/model/api/INodeReference.kt

@@ -64,7 +64,7 @@ fun INodeReference.toSerialized(): NodeReference = if (this is NodeReference) th
 class NodeReferenceKSerializer : KSerializer<INodeReference> {
     override fun deserialize(decoder: Decoder): INodeReference {
         val serialized = decoder.decodeString()
-        return INodeReferenceSerializer.tryDeserialize(serialized) ?: SerializedNodeReference(serialized)
+        return NodeReference(serialized)
     }
 
     override val descriptor: SerialDescriptor = PrimitiveSerialDescriptor("modelix.INodeReference", PrimitiveKind.STRING)