Merge pull request #279 from modelix/fix/bulk-sync-import-fix

MODELIX-561 Import into model-server is unsuccessful

Author: mhuster23

bulk-model-sync-gradle-test/src/test/kotlin/org/modelix/model/sync/bulk/gradle/test/PushTest.kt

@@ -10,16 +10,17 @@ import org.modelix.metamodel.TypedLanguagesRegistry
 import org.modelix.metamodel.typed
 import org.modelix.model.ModelFacade
 import org.modelix.model.api.ConceptReference
-import org.modelix.model.api.IBranch
 import org.modelix.model.api.getDescendants
 import org.modelix.model.api.getRootNode
+import org.modelix.model.client2.IModelClientV2
 import org.modelix.model.client2.ModelClientV2PlatformSpecificBuilder
 import org.modelix.model.client2.getReplicatedModel
+import org.modelix.model.client2.runWrite
 import org.modelix.model.data.ModelData
 import org.modelix.model.data.NodeData
+import org.modelix.model.lazy.BranchReference
 import org.modelix.model.lazy.RepositoryId
 import org.modelix.model.server.Main
-import org.modelix.model.sleep
 import org.modelix.model.sync.bulk.asExported
 import java.io.File
 import kotlin.test.assertContentEquals
@@ -40,32 +41,33 @@ class PushTest {
         val repoId = RepositoryId("ci-test")
         val branchName = "master"
         val url = "http://0.0.0.0:${Main.DEFAULT_PORT}/v2"
+
         val branchRef = ModelFacade.createBranchReference(repoId, branchName)
-        val client = ModelClientV2PlatformSpecificBuilder().url(url).build()
+        val client = ModelClientV2PlatformSpecificBuilder().url(url).build().apply { runBlocking { init() } }
+        val replicatedModel = client.getReplicatedModel(branchRef)
+        val branch = runBlocking { replicatedModel.start() }
 
-        val branch = runBlocking {
-            client.init()
-            client.getReplicatedModel(branchRef).start()
-        }
         branch.runRead {
             assertContentEquals(inputModel.root.children, branch.getRootNode().allChildren.map { it.asExported() })
         }
+        replicatedModel.dispose()
 
-        applyChangesForPullTest(branch)
+        applyChangesForPullTest(client, branchRef)
     }
 
-    private fun applyChangesForPullTest(branch: IBranch) {
-        branch.runWrite {
-            val graphNodes = branch.getRootNode()
-                .getDescendants(false)
-                .filter { it.getConceptReference() == ConceptReference(_C_UntypedImpl_Node.getUID()) }
-                .map { it.typed<N_Node>() }
-                .toList()
+    private fun applyChangesForPullTest(client: IModelClientV2, branchRef: BranchReference) {
+        runBlocking {
+            client.runWrite(branchRef) { rootNode ->
+                val graphNodes = rootNode
+                    .getDescendants(false)
+                    .filter { it.getConceptReference() == ConceptReference(_C_UntypedImpl_Node.getUID()) }
+                    .map { it.typed<N_Node>() }
+                    .toList()
 
-            graphNodes[0].name = "X"
-            graphNodes[1].name = "Y"
-            graphNodes[2].name = "Z"
+                graphNodes[0].name = "X"
+                graphNodes[1].name = "Y"
+                graphNodes[2].name = "Z"
+            }
         }
-        sleep(5000) // changes are pushed asynchronously to the server. wait for the propagation
     }
 }

bulk-model-sync-gradle/src/main/kotlin/org/modelix/model/sync/bulk/gradle/tasks/ImportIntoModelServer.kt

@@ -30,9 +30,8 @@ import org.gradle.api.tasks.TaskAction
 import org.modelix.model.ModelFacade
 import org.modelix.model.api.ILanguage
 import org.modelix.model.api.ILanguageRepository
-import org.modelix.model.api.getRootNode
 import org.modelix.model.client2.ModelClientV2PlatformSpecificBuilder
-import org.modelix.model.client2.getReplicatedModel
+import org.modelix.model.client2.runWrite
 import org.modelix.model.lazy.RepositoryId
 import org.modelix.model.sync.bulk.ModelImporter
 import org.modelix.model.sync.bulk.importFilesAsRootChildren
@@ -67,20 +66,17 @@ abstract class ImportIntoModelServer @Inject constructor(of: ObjectFactory) : De
 
         val branchRef = ModelFacade.createBranchReference(repoId, branchName.get())
         val client = ModelClientV2PlatformSpecificBuilder().url(url.get()).build()
-        val branch = runBlocking {
-            client.init()
-            client.getReplicatedModel(branchRef).start()
-        }
-
         val files = inputDir.listFiles()?.filter { it.extension == "json" }
         if (files.isNullOrEmpty()) error("no json files found")
 
-        branch.runWrite {
-            val rootNode = branch.getRootNode()
-            logger.info("Got root node: {}", rootNode)
-            logger.info("Importing...")
-            ModelImporter(branch.getRootNode()).importFilesAsRootChildren(*files.toTypedArray())
-            logger.info("Import finished")
+        runBlocking {
+            client.init()
+            client.runWrite(branchRef) { rootNode ->
+                logger.info("Got root node: {}", rootNode)
+                logger.info("Importing...")
+                ModelImporter(rootNode).importFilesAsRootChildren(files)
+                logger.info("Import finished")
+            }
         }
     }
 }

bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/ModelImporter.kt

@@ -53,6 +53,7 @@ class ModelImporter(private val root: INode) {
      */
     @JvmName("importData")
     fun import(data: ModelData) {
+        logImportSize(data.root, logger)
         logger.info { "Building indices for import..." }
         originalIdToExisting.clear()
         postponedReferences.clear()

bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/Util.kt

@@ -16,9 +16,54 @@
 
 package org.modelix.model.sync.bulk
 
+import mu.KLogger
+import org.modelix.model.api.BuiltinLanguages
 import org.modelix.model.data.ModelData
 import org.modelix.model.data.NodeData
 
+fun mergeModelData(models: Collection<ModelData>): ModelData {
+    return ModelData(root = NodeData(children = models.map { it.root }))
+}
+
+@Deprecated("use collection parameter for better performance")
 fun mergeModelData(vararg models: ModelData): ModelData {
     return ModelData(root = NodeData(children = models.map { it.root }))
 }
+
+internal fun logImportSize(nodeData: NodeData, logger: KLogger) {
+    logger.debug { measureImportSize(nodeData).toString() }
+}
+
+private data class ImportSizeMetrics(
+    var numModules: Int = 0,
+    var numModels: Int = 0,
+    val concepts: MutableSet<String> = mutableSetOf(),
+    var numProperties: Int = 0,
+    var numReferences: Int = 0,
+) {
+    override fun toString(): String {
+        return """
+            [Bulk Model Sync Import Size]
+            number of modules: $numModules
+            number of models: $numModels
+            number of concepts: ${concepts.size}
+            number of properties: $numProperties
+            number of references: $numReferences
+        """.trimIndent()
+    }
+}
+
+private fun measureImportSize(data: NodeData, metrics: ImportSizeMetrics = ImportSizeMetrics()): ImportSizeMetrics {
+    data.concept?.let { metrics.concepts.add(it) }
+
+    when (data.concept) {
+        BuiltinLanguages.MPSRepositoryConcepts.Module.getUID() -> metrics.numModules++
+        BuiltinLanguages.MPSRepositoryConcepts.Model.getUID() -> metrics.numModels++
+    }
+
+    metrics.numProperties += data.properties.size
+    metrics.numReferences += data.references.size
+
+    data.children.forEach { measureImportSize(it, metrics) }
+    return metrics
+}

bulk-model-sync-lib/src/jvmMain/kotlin/org/modelix/model/sync/bulk/PlatformSpecific.kt

@@ -35,6 +35,12 @@ fun ModelImporter.import(jsonFile: File) {
     import(data)
 }
 
+fun ModelImporter.importFilesAsRootChildren(files: Collection<File>) {
+    val models = files.map { ModelData.fromJson(it.readText()) }
+    import(mergeModelData(models))
+}
+
+@Deprecated("use collection parameter for better performance")
 fun ModelImporter.importFilesAsRootChildren(vararg files: File) {
     val models = files.map { ModelData.fromJson(it.readText()) }
     import(mergeModelData(*models.toTypedArray()))

model-client/src/commonMain/kotlin/org/modelix/model/client2/ModelClientV2.kt

@@ -34,13 +34,18 @@ import kotlinx.coroutines.launch
 import kotlinx.serialization.json.Json
 import org.modelix.model.IVersion
 import org.modelix.model.api.IIdGenerator
+import org.modelix.model.api.INode
 import org.modelix.model.api.IdGeneratorDummy
+import org.modelix.model.api.TreePointer
+import org.modelix.model.api.getRootNode
 import org.modelix.model.client.IdGenerator
 import org.modelix.model.lazy.BranchReference
+import org.modelix.model.lazy.CLTree
 import org.modelix.model.lazy.CLVersion
 import org.modelix.model.lazy.ObjectStoreCache
 import org.modelix.model.lazy.RepositoryId
 import org.modelix.model.lazy.computeDelta
+import org.modelix.model.operations.OTBranch
 import org.modelix.model.persistent.HashUtil
 import org.modelix.model.persistent.MapBasedStore
 import org.modelix.model.server.api.v2.VersionDelta
@@ -307,3 +312,25 @@ private fun URLBuilder.appendPathSegmentsEncodingSlash(vararg components: String
 }
 
 fun VersionDelta.getAllObjects(): Map<String, String> = objectsMap + objects.associateBy { HashUtil.sha256(it) }
+
+/**
+ * Performs a write transaction on the root node of the given branch.
+ */
+suspend fun <T> IModelClientV2.runWrite(branchRef: BranchReference, body: (INode) -> T): T {
+    val client = this
+    val baseVersion = client.pull(branchRef, null)
+    val branch = OTBranch(TreePointer(baseVersion.getTree(), client.getIdGenerator()), client.getIdGenerator(), (client as ModelClientV2).store)
+    val result = branch.computeWrite {
+        body(branch.getRootNode())
+    }
+    val (ops, newTree) = branch.getPendingChanges()
+    val newVersion = CLVersion.createRegularVersion(
+        id = client.getIdGenerator().generate(),
+        author = client.getUserId(),
+        tree = newTree as CLTree,
+        baseVersion = baseVersion as CLVersion?,
+        operations = ops.map { it.getOriginalOp() }.toTypedArray(),
+    )
+    client.push(branchRef, newVersion, baseVersion)
+    return result
+}

model-client/src/commonMain/kotlin/org/modelix/model/client2/ReplicatedModel.kt

@@ -21,6 +21,9 @@ import org.modelix.model.lazy.CLTree
 import org.modelix.model.lazy.CLVersion
 import org.modelix.model.operations.OTBranch
 
+/**
+ * Dispose should be called on this, as otherwise a regular polling will go on.
+ */
 class ReplicatedModel(
     val client: IModelClientV2,
     val branchRef: BranchReference,