fix(mps-sync-plugin): several fixes after trying to import MPS-extensions

slisson · slisson · commit 69dc4525fed8 · 2025-04-05T15:15:51.000+02:00
diff --git a/datastructures/src/commonMain/kotlin/org/modelix/datastructures/btree/BTreeNode.kt b/datastructures/src/commonMain/kotlin/org/modelix/datastructures/btree/BTreeNode.kt
@@ -3,6 +3,8 @@ package org.modelix.datastructures.btree
 import org.modelix.datastructures.objects.IObjectData
 import org.modelix.datastructures.objects.IObjectDeserializer
 import org.modelix.datastructures.objects.IObjectReferenceFactory
+import org.modelix.datastructures.objects.Object
+import org.modelix.datastructures.objects.getDescendantsAndSelf
 import org.modelix.datastructures.serialization.SerializationSeparators
 import org.modelix.streams.IStream
 
@@ -33,6 +35,11 @@ sealed class BTreeNode<K, V> : IObjectData {
     fun splitIfNecessary(): Replacement<K, V> = if (isOverfilled()) split() else Replacement.Single(this)
     abstract fun mergeWithSibling(knownSeparator: K, right: BTreeNode<K, V>): BTreeNode<K, V>
 
+    override fun objectDiff(self: Object<*>, oldObject: Object<*>?): IStream.Many<Object<*>> {
+        // TODO performance
+        return self.getDescendantsAndSelf()
+    }
+
     class Deserializer<K, V>(val config: BTreeConfig<K, V>) : IObjectDeserializer<BTreeNode<K, V>> {
         override fun deserialize(
             serialized: String,
diff --git a/datastructures/src/commonMain/kotlin/org/modelix/datastructures/objects/IObjectData.kt b/datastructures/src/commonMain/kotlin/org/modelix/datastructures/objects/IObjectData.kt
@@ -19,10 +19,7 @@ interface IObjectData {
      * Callers should compare the hashes of the old and new object before calling this method.
      * It assumes that the [oldObject] is different from this one and calling it anyway will result in a bigger diff.
      */
-    fun objectDiff(self: Object<*>, oldObject: Object<*>?): IStream.Many<Object<*>> {
-        requireDifferentHash(oldObject?.data)
-        return self.getDescendantsAndSelf()
-    }
+    fun objectDiff(self: Object<*>, oldObject: Object<*>?): IStream.Many<Object<*>>
 }
 
 @Deprecated("Just don't use it and assume they are different")
diff --git a/datastructures/src/commonMain/kotlin/org/modelix/datastructures/patricia/PatriciaNode.kt b/datastructures/src/commonMain/kotlin/org/modelix/datastructures/patricia/PatriciaNode.kt
@@ -12,6 +12,8 @@ import org.modelix.datastructures.objects.IObjectGraph
 import org.modelix.datastructures.objects.IObjectReferenceFactory
 import org.modelix.datastructures.objects.Object
 import org.modelix.datastructures.objects.ObjectReference
+import org.modelix.datastructures.objects.asObject
+import org.modelix.datastructures.objects.getDescendantsAndSelf
 import org.modelix.datastructures.objects.getHashString
 import org.modelix.datastructures.objects.upcast
 import org.modelix.datastructures.serialization.SplitJoinSerializer
@@ -315,6 +317,60 @@ data class PatriciaNode<K, V : Any>(
         }
     }
 
+    override fun objectDiff(self: Object<*>, oldObject: Object<*>?): IStream.Many<Object<*>> {
+        return objectDiff(self, oldObject, "")
+    }
+
+    fun objectDiff(self: Object<*>, oldObject: Object<*>?, path: CharSequence): IStream.Many<Object<*>> {
+        if (oldObject == null) {
+            return self.getDescendantsAndSelf()
+        }
+        self as Object<PatriciaNode<K, V>>
+        oldObject as Object<PatriciaNode<K, V>>
+        val oldNode: PatriciaNode<K, V> = oldObject.data as PatriciaNode<K, V>
+        return if (ownPrefix == oldNode.ownPrefix) {
+            val pathForChildren = path + ownPrefix
+            val matchingChildren = if (firstChars == oldNode.firstChars) {
+                children.zip(oldNode.children)
+            } else {
+                val newChildren = firstChars.asSequence().zip(children.asSequence()).toMap()
+                val oldChildren = oldNode.firstChars.asSequence().zip(oldNode.children.asSequence()).toMap()
+                val allFirstChars = newChildren.keys.plus(oldChildren.keys).distinct()
+                allFirstChars.map { newChildren[it] to oldChildren[it] }
+            }
+            val changesFromChildren = IStream.many(matchingChildren).flatMap { (newChildRef, oldChildRef) ->
+                if (newChildRef == null) {
+                    IStream.empty()
+                } else {
+                    if (oldChildRef == null) {
+                        newChildRef.resolve().flatMap { it.getDescendantsAndSelf() }
+                    } else {
+                        if (newChildRef.getHash() == oldChildRef.getHash()) {
+                            IStream.empty()
+                        } else {
+                            newChildRef.resolve().zipWith(oldChildRef.resolve()) { newChild, oldChild ->
+                                newChild.data.objectDiff(newChild, oldChild, pathForChildren)
+                            }.flatten()
+                        }
+                    }
+                }
+            }
+
+            IStream.of(self) + changesFromChildren
+        } else {
+            val commonPrefix = ownPrefix.commonPrefixWith(oldNode.ownPrefix)
+            self.split(commonPrefix).let {
+                it.data.objectDiff(it, oldObject.split(commonPrefix), path).filter { it.graph == config.graph }
+            }
+        }
+    }
+
+    private fun <K, V : Any> Object<PatriciaNode<K, V>>.split(commonPrefix: CharSequence): Object<PatriciaNode<K, V>> {
+        val splitted = data.split(commonPrefix)
+        if (splitted === data) return this
+        return splitted.asObject(IObjectGraph.FREE_FLOATING)
+    }
+
     class Deserializer<K, V : Any>(val config: (IObjectGraph) -> PatriciaTrieConfig<K, V>) : IObjectDeserializer<PatriciaNode<K, V>> {
         constructor(config: PatriciaTrieConfig<K, V>) : this({ config })
         override fun deserialize(
diff --git a/model-datastructure/src/commonMain/kotlin/org/modelix/datastructures/model/IdTranslatingModelTree.kt b/model-datastructure/src/commonMain/kotlin/org/modelix/datastructures/model/IdTranslatingModelTree.kt
@@ -176,12 +176,18 @@ abstract class IdTranslatingModelTree<ExternalId, InternalId>(val tree: IGeneric
 
 @JvmName("withIdTranslationToInt64")
 fun IGenericModelTree<INodeReference>.withIdTranslation(): IGenericModelTree<Long> {
-    return NodeReferenceAsLongModelTree(this)
+    return when (this) {
+        is LongAsNodeReferenceModelTree -> this.tree
+        else -> NodeReferenceAsLongModelTree(this)
+    }
 }
 
 @JvmName("withIdTranslationToNodeReferences")
 fun IGenericModelTree<Long>.withIdTranslation(): IGenericModelTree<INodeReference> {
-    return LongAsNodeReferenceModelTree(this)
+    return when (this) {
+        is NodeReferenceAsLongModelTree -> this.tree
+        else -> LongAsNodeReferenceModelTree(this)
+    }
 }
 
 fun INodeReference.extractInt64Id(expectedTree: TreeId): Long {
diff --git a/model-datastructure/src/commonMain/kotlin/org/modelix/datastructures/model/NodeObjectData.kt b/model-datastructure/src/commonMain/kotlin/org/modelix/datastructures/model/NodeObjectData.kt
@@ -9,6 +9,7 @@ import org.modelix.datastructures.objects.IObjectData
 import org.modelix.datastructures.objects.IObjectDeserializer
 import org.modelix.datastructures.objects.IObjectReferenceFactory
 import org.modelix.datastructures.objects.LongDataTypeConfiguration
+import org.modelix.datastructures.objects.Object
 import org.modelix.datastructures.objects.ObjectReference
 import org.modelix.datastructures.objects.asKSerializer
 import org.modelix.datastructures.serialization.SplitJoinFormat
@@ -23,6 +24,7 @@ import org.modelix.model.api.IReadableNode
 import org.modelix.model.api.IReferenceLinkReference
 import org.modelix.model.api.NullChildLinkReference
 import org.modelix.model.api.meta.NullConcept
+import org.modelix.streams.IStream
 
 @Serializable
 data class NodeObjectData<NodeId>(
@@ -109,6 +111,10 @@ data class NodeObjectData<NodeId>(
         return copy(children = children.filterNot { deserializer!!.nodeIdTypeConfig.equal(it, childId) })
     }
 
+    override fun objectDiff(self: Object<*>, oldObject: Object<*>?): IStream.Many<Object<*>> {
+        return IStream.of(self)
+    }
+
     class Deserializer<NodeId>(
         val nodeIdTypeConfig: IDataTypeConfiguration<NodeId>,
         val treeId: TreeId,
diff --git a/model-datastructure/src/commonMain/kotlin/org/modelix/model/mutable/VersionedModelTree.kt b/model-datastructure/src/commonMain/kotlin/org/modelix/model/mutable/VersionedModelTree.kt
@@ -171,12 +171,18 @@ class VersionedModelTree(
         override fun mutate(parameters: MutationParameters<INodeReference>) {
             when (parameters) {
                 is MutationParameters.AddNew<INodeReference> -> {
-                    apply(
-                        AddNewChildrenOp(
-                            position = PositionInRole(parameters.nodeId, parameters.role, parameters.index),
-                            childIdsAndConcepts = parameters.newIdAndConcept.toList(),
-                        ),
-                    )
+                    // split operation into smaller chunks to avoid large operation objects
+                    var indexOffset = 0
+                    for (chunk in parameters.newIdAndConcept.asSequence().chunked(100)) {
+                        val index = if (parameters.index >= 0) parameters.index + indexOffset else parameters.index
+                        apply(
+                            AddNewChildrenOp(
+                                position = PositionInRole(parameters.nodeId, parameters.role, index),
+                                childIdsAndConcepts = chunk,
+                            ),
+                        )
+                        indexOffset += chunk.size
+                    }
                 }
                 is MutationParameters.Move<INodeReference> -> {
                     val targetPosition = PositionInRole(parameters.nodeId, parameters.role, parameters.index)
diff --git a/model-datastructure/src/commonMain/kotlin/org/modelix/model/persistent/CPNode.kt b/model-datastructure/src/commonMain/kotlin/org/modelix/model/persistent/CPNode.kt
@@ -3,6 +3,7 @@ package org.modelix.model.persistent
 import org.modelix.datastructures.objects.IObjectData
 import org.modelix.datastructures.objects.IObjectDeserializer
 import org.modelix.datastructures.objects.IObjectReferenceFactory
+import org.modelix.datastructures.objects.Object
 import org.modelix.datastructures.objects.ObjectReference
 import org.modelix.model.lazy.COWArrays.copy
 import org.modelix.model.lazy.COWArrays.insert
@@ -14,6 +15,7 @@ import org.modelix.model.persistent.SerializationUtil.escape
 import org.modelix.model.persistent.SerializationUtil.longFromHex
 import org.modelix.model.persistent.SerializationUtil.longToHex
 import org.modelix.model.persistent.SerializationUtil.unescape
+import org.modelix.streams.IStream
 import kotlin.jvm.JvmStatic
 
 class CPNode(
@@ -52,6 +54,10 @@ class CPNode(
         return sb.toString()
     }
 
+    override fun objectDiff(self: Object<*>, oldObject: Object<*>?): IStream.Many<Object<*>> {
+        return IStream.of(self)
+    }
+
     fun getChildrenIds(): Iterable<Long> {
         return Iterable { childrenIds.iterator() }
     }
diff --git a/model-datastructure/src/commonMain/kotlin/org/modelix/model/persistent/CPVersion.kt b/model-datastructure/src/commonMain/kotlin/org/modelix/model/persistent/CPVersion.kt
@@ -4,6 +4,7 @@ import org.modelix.datastructures.hamt.HamtNode
 import org.modelix.datastructures.objects.IObjectData
 import org.modelix.datastructures.objects.IObjectDeserializer
 import org.modelix.datastructures.objects.IObjectReferenceFactory
+import org.modelix.datastructures.objects.Object
 import org.modelix.datastructures.objects.ObjectReference
 import org.modelix.datastructures.objects.getHashString
 import org.modelix.datastructures.patricia.PatriciaNode
@@ -14,6 +15,7 @@ import org.modelix.model.persistent.SerializationUtil.escape
 import org.modelix.model.persistent.SerializationUtil.longFromHex
 import org.modelix.model.persistent.SerializationUtil.longToHex
 import org.modelix.model.persistent.SerializationUtil.unescape
+import org.modelix.streams.IStream
 
 data class CPVersion(
     val id: Long,
@@ -230,4 +232,8 @@ data class CPVersion(
             originalVersion = originalVersion?.asUnloaded(),
         )
     }
+
+    override fun objectDiff(self: Object<*>, oldObject: Object<*>?): IStream.Many<Object<*>> {
+        throw UnsupportedOperationException("CLVersion.diff")
+    }
 }
diff --git a/model-datastructure/src/commonMain/kotlin/org/modelix/model/persistent/OperationsList.kt b/model-datastructure/src/commonMain/kotlin/org/modelix/model/persistent/OperationsList.kt
@@ -3,7 +3,9 @@ package org.modelix.model.persistent
 import org.modelix.datastructures.objects.IObjectData
 import org.modelix.datastructures.objects.IObjectDeserializer
 import org.modelix.datastructures.objects.IObjectReferenceFactory
+import org.modelix.datastructures.objects.Object
 import org.modelix.datastructures.objects.ObjectReference
+import org.modelix.datastructures.objects.getDescendantsAndSelf
 import org.modelix.datastructures.objects.getHashString
 import org.modelix.model.operations.IOperation
 import org.modelix.streams.IStream
@@ -44,6 +46,10 @@ abstract class OperationsList() : IObjectData {
     }
 
     abstract fun getOperations(): IStream.Many<IOperation>
+
+    override fun objectDiff(self: Object<*>, oldObject: Object<*>?): IStream.Many<Object<*>> {
+        return self.getDescendantsAndSelf()
+    }
 }
 
 class LargeOperationsList(val subLists: Array<out ObjectReference<OperationsList>>) : OperationsList() {
diff --git a/model-server/src/main/resources/org/modelix/model/server/store/ignite.xml b/model-server/src/main/resources/org/modelix/model/server/store/ignite.xml
@@ -41,11 +41,13 @@
     <bean id="ignite.cfg" class="org.apache.ignite.configuration.IgniteConfiguration">
         <property name="dataStorageConfiguration">
             <bean class="org.apache.ignite.configuration.DataStorageConfiguration">
+                <property name="pageSize" value="16384" />
                 <property name="defaultDataRegionConfiguration">
                     <bean class="org.apache.ignite.configuration.DataRegionConfiguration">
                         <property name="maxSize" value="${ignite.cache.size}"/>
                         <property name="pageEvictionMode" value="RANDOM_LRU" />
-                        <property name="emptyPagesPoolSize" value="2000" />
+                        <property name="emptyPagesPoolSize" value="630" />
+                        <property name="evictionThreshold" value="0.8" />
                     </bean>
                 </property>
             </bean>
diff --git a/mps-model-adapters-plugin/src/test/kotlin/org/modelix/model/mpsadapters/MPSModuleAsNodeTest.kt b/mps-model-adapters-plugin/src/test/kotlin/org/modelix/model/mpsadapters/MPSModuleAsNodeTest.kt
@@ -8,7 +8,7 @@ class MPSModuleAsNodeTest : MpsAdaptersTestBase("SimpleProject") {
     fun `test resolve language dependency from reference`() {
         val repositoryNode: INode = mpsProject.repository.asLegacyNode()
         val languageDependencyNodeReference = NodeReference(
-            "mps-lang:f3061a53-9226-4cc5-a443-f952ceaf5816#IN#mps-module:6517ba0d-f632-49c5-a166-401587c2c3ca()",
+            "mps-lang:f3061a53-9226-4cc5-a443-f952ceaf5816#IN#mps-module:6517ba0d-f632-49c5-a166-401587c2c3ca",
         )
 
         val resolvedLanguageDependency = readAction {
diff --git a/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSArea.kt b/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSArea.kt
@@ -3,7 +3,6 @@ package org.modelix.model.mpsadapters
 import jetbrains.mps.ide.ThreadUtils
 import jetbrains.mps.project.ProjectBase
 import jetbrains.mps.project.facets.JavaModuleFacet
-import jetbrains.mps.project.structure.modules.ModuleReference
 import jetbrains.mps.smodel.GlobalModelAccess
 import jetbrains.mps.smodel.ModelImports
 import org.jetbrains.mps.openapi.model.SNodeReference
@@ -149,27 +148,11 @@ data class MPSArea(val repository: SRepository) : IArea, IAreaReference {
     }
 
     private fun resolveMPSModuleReference(ref: INodeReference): MPSModuleAsNode<*>? {
-        val moduleRef = if (ref is MPSModuleReference) {
-            ref.moduleReference
-        } else {
-            val serializedRef = ref.serialize().substringAfter("${MPSModuleReference.PREFIX}:")
-            ModuleReference.parseReference(serializedRef)
-        }
-
-        return moduleRef.resolve(repository)?.let { MPSModuleAsNode(it) }
+        return MPSModuleReference.tryConvert(ref)?.moduleReference?.resolve(repository)?.let { MPSModuleAsNode(it) }
     }
 
     private fun resolveMPSNodeReference(ref: INodeReference): INode? {
-        if (ref is MPSNodeReference) {
-            return resolveSNodeReferenceToMPSNode(ref.ref)
-        }
-        val serialized = ref.serialize()
-        val serializedMPSRef = when {
-            serialized.startsWith("mps-node:") -> serialized.substringAfter("mps-node:")
-            serialized.startsWith("mps:") -> serialized.substringAfter("mps:")
-            else -> return null
-        }
-        return resolveSNodeReferenceToMPSNode(MPSReferenceParser.parseSNodeReference(serializedMPSRef))
+        return MPSNodeReference.tryConvert(ref)?.ref?.let { resolveSNodeReferenceToMPSNode(it) }
     }
 
     private fun resolveSNodeReferenceToMPSNode(sNodeReference: SNodeReference): INode? {
@@ -208,7 +191,7 @@ data class MPSArea(val repository: SRepository) : IArea, IAreaReference {
         } else {
             val serialized = ref.serialize()
             val serializedModuleRef = serialized.substringAfter("${MPSJavaModuleFacetReference.PREFIX}:")
-            ModuleReference.parseReference(serializedModuleRef)
+            MPSReferenceParser.parseSModuleReference(serializedModuleRef)
         }
 
         val facet = moduleRef.resolve(repository)?.getFacetOfType(JavaModuleFacet.FACET_TYPE)
@@ -254,7 +237,7 @@ data class MPSArea(val repository: SRepository) : IArea, IAreaReference {
             ref.userModuleReference
         } else {
             val serializedModuleRef = serialized.substringAfter(MPSModuleDependencyReference.SEPARATOR)
-            ModuleReference.parseReference(serializedModuleRef)
+            MPSReferenceParser.parseSModuleReference(serializedModuleRef)
         }
 
         return userModuleReference.resolve(repository)
@@ -297,7 +280,7 @@ data class MPSArea(val repository: SRepository) : IArea, IAreaReference {
             val serializedModuleRef = serialized
                 .substringAfter("${MPSProjectModuleReference.PREFIX}:")
                 .substringBefore(MPSProjectModuleReference.SEPARATOR)
-            ModuleReference.parseReference(serializedModuleRef)
+            MPSReferenceParser.parseSModuleReference(serializedModuleRef)
         }
 
         val projectRef = if (ref is MPSProjectModuleReference) {
diff --git a/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSReferenceParser.kt b/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSReferenceParser.kt
@@ -5,6 +5,7 @@ import jetbrains.mps.util.StringUtil
 import org.jetbrains.mps.openapi.model.SModelReference
 import org.jetbrains.mps.openapi.model.SNodeId
 import org.jetbrains.mps.openapi.model.SNodeReference
+import org.jetbrains.mps.openapi.module.SModuleReference
 import org.jetbrains.mps.openapi.persistence.PersistenceFacade
 import org.jetbrains.mps.openapi.persistence.PersistenceFacade.IncorrectModelReferenceFormatException
 import org.jetbrains.mps.openapi.persistence.PersistenceFacade.IncorrectNodeIdFormatException
@@ -13,6 +14,15 @@ import org.jetbrains.mps.openapi.persistence.PersistenceFacade.IncorrectNodeIdFo
  * Implementations in MPS expect a model name to be present in the reference.
  */
 object MPSReferenceParser {
+    fun parseSModuleReference(serialized: String): SModuleReference {
+        return if (serialized.contains('(') && serialized.contains(')')) {
+            PersistenceFacade.getInstance().createModuleReference(serialized)
+        } else {
+            val moduleId = PersistenceFacade.getInstance().createModuleId(serialized)
+            PersistenceFacade.getInstance().createModuleReference(moduleId, null)
+        }
+    }
+
     fun parseSModelReference(serialized: String): SModelReference {
         val modelRefParts = parseReferenceInternal(serialized)
         val pf = PersistenceFacade.getInstance()
diff --git a/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSReferences.kt b/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSReferences.kt
diff --git a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/ui/ModelSyncStatusWidget.kt b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/ui/ModelSyncStatusWidget.kt
diff --git a/streams/src/commonMain/kotlin/org/modelix/streams/BulkRequestStreamExecutor.kt b/streams/src/commonMain/kotlin/org/modelix/streams/BulkRequestStreamExecutor.kt
