Merge pull request #140 from modelix/feature/model-sync-lib

MODELIX-447 model-sync-lib: New component for exporting/importing from model-api to JSON

Author: slisson

commitlint.config.js

@@ -14,6 +14,7 @@ module.exports = {
         "model-api",
         "model-client",
         "model-server",
+        "model-sync-lib",
         "ts-model-api",
       ],
     ],

model-api/src/commonMain/kotlin/org/modelix/model/api/INode.kt

@@ -69,10 +69,11 @@ interface INode {
     val allChildren: Iterable<INode>
 
     /**
-     * Moves a child node to the given role and index.
+     * Moves a node to this node's children with the given role and index.
+     * The child node can originate from a different parent.
      *
      * @param role target role
-     * @param index target index
+     * @param index target index within the role
      * @param child child node to be moved
      */
     fun moveChild(role: String?, index: Int, child: INode)
@@ -97,7 +98,7 @@ interface INode {
      * Creates and adds a new child node to this node at the specified index.
      *
      * @param role role, where the node should be added
-     * @param index index, where the node should be added
+     * @param index index within the role, where the node should be added
      * @param concept reference to a concept, of which the new node is instance of
      * @return new child node
      */

model-api/src/commonMain/kotlin/org/modelix/model/data/ModelData.kt

@@ -1,7 +1,6 @@
 package org.modelix.model.data
 
 import kotlinx.serialization.Serializable
-import kotlinx.serialization.decodeFromString
 import kotlinx.serialization.encodeToString
 import kotlinx.serialization.json.Json
 import org.modelix.model.api.*
@@ -19,6 +18,10 @@ data class ModelData(
             val createdNodes = HashMap<String, Long>()
             val pendingReferences = ArrayList<() -> Unit>()
             val parentId = ITree.ROOT_ID
+            if (root.id != null) {
+                createdNodes[root.id] = parentId
+            }
+            setOriginalId(root, t, parentId)
             for (nodeData in root.children) {
                 loadNode(nodeData, t, parentId, createdNodes, pendingReferences)
             }
@@ -37,6 +40,7 @@ data class ModelData(
         val createdId = t.addNewChild(parentId, nodeData.role, -1, conceptRef)
         if (nodeData.id != null) {
             createdNodes[nodeData.id] = createdId
+            setOriginalId(nodeData, t, createdId)
         }
         for (propertyData in nodeData.properties) {
             t.setProperty(createdId, propertyData.key, propertyData.value)
@@ -52,6 +56,15 @@ data class ModelData(
         }
     }
 
+    private fun setOriginalId(
+        nodeData: NodeData,
+        t: IWriteTransaction,
+        nodeId: Long
+    ) {
+        val key = NodeData.idPropertyKey
+        t.setProperty(nodeId, key, nodeData.properties[key] ?: nodeData.id)
+    }
+
     companion object {
         private val prettyJson = Json { prettyPrint = true }
         fun fromJson(serialized: String): ModelData = Json.decodeFromString(serialized)
@@ -66,7 +79,11 @@ data class NodeData(
     val children: List<NodeData> = emptyList(),
     val properties: Map<String, String> = emptyMap(),
     val references: Map<String, String> = emptyMap()
-)
+) {
+    companion object {
+        const val idPropertyKey = "#mpsNodeId#"
+    }
+}
 
 fun NodeData.uid(model: ModelData): String {
     return (model.id ?: throw IllegalArgumentException("Model has no ID")) +
@@ -84,6 +101,6 @@ fun INode.asData(): NodeData = NodeData(
     children = allChildren.map { it.asData() }
 )
 
-public inline fun <K, V : Any> Iterable<K>.associateWithNotNull(valueSelector: (K) -> V?): Map<K, V> {
+inline fun <K, V : Any> Iterable<K>.associateWithNotNull(valueSelector: (K) -> V?): Map<K, V> {
     return associateWith { valueSelector(it) }.filterValues { it != null } as Map<K, V>
 }

model-client/src/commonMain/kotlin/org/modelix/model/operations/MoveNodeOp.kt

@@ -48,6 +48,10 @@ class MoveNodeOp(val childId: Long, val targetPosition: PositionInRole) : Abstra
         override fun invert(): List<IOperation> {
             return listOf(MoveNodeOp(childId, sourcePosition))
         }
+
+        override fun toString(): String {
+            return "applied:MoveNodeOp ${childId.toString(16)}: $sourcePosition->$targetPosition"
+        }
     }
 
     override fun captureIntend(tree: ITree, store: IDeserializingKeyValueStore): IOperationIntend {

model-sync-lib/.gitignore

@@ -0,0 +1,42 @@
+.gradle
+build/
+!gradle/wrapper/gradle-wrapper.jar
+!**/src/main/**/build/
+!**/src/test/**/build/
+
+### IntelliJ IDEA ###
+.idea/modules.xml
+.idea/jarRepositories.xml
+.idea/compiler.xml
+.idea/libraries/
+*.iws
+*.iml
+*.ipr
+out/
+!**/src/main/**/out/
+!**/src/test/**/out/
+
+### Eclipse ###
+.apt_generated
+.classpath
+.factorypath
+.project
+.settings
+.springBeans
+.sts4-cache
+bin/
+!**/src/main/**/bin/
+!**/src/test/**/bin/
+
+### NetBeans ###
+/nbproject/private/
+/nbbuild/
+/dist/
+/nbdist/
+/.nb-gradle/
+
+### VS Code ###
+.vscode/
+
+### Mac OS ###
+.DS_Store

model-sync-lib/build.gradle.kts

@@ -0,0 +1,41 @@
+plugins {
+    alias(libs.plugins.kotlin.multiplatform)
+}
+
+repositories {
+    mavenCentral()
+}
+
+kotlin {
+    jvm {
+        jvmToolchain(11)
+        testRuns["test"].executionTask.configure {
+            useJUnitPlatform()
+        }
+    }
+    sourceSets {
+        val commonMain by getting {
+            dependencies {
+                implementation(project(":model-api"))
+                implementation(libs.kotlin.serialization.json)
+            }
+        }
+
+        val commonTest by getting {
+            dependencies {
+                implementation(project(":model-api"))
+                implementation(libs.kotlin.serialization.json)
+                implementation(project(":model-client", configuration = "jvmRuntimeElements"))
+                implementation(kotlin("test"))
+            }
+        }
+    }
+}
+
+publishing {
+    publications {
+        create<MavenPublication>("maven") {
+            from(components["kotlin"])
+        }
+    }
+}

model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/ModelExporter.kt

@@ -0,0 +1,31 @@
+package org.modelix.model.sync
+
+import org.modelix.model.api.INode
+import org.modelix.model.api.serialize
+import org.modelix.model.data.ModelData
+import org.modelix.model.data.NodeData
+import org.modelix.model.data.associateWithNotNull
+import java.io.File
+
+class ModelExporter(private val root: INode) {
+
+    fun export(outputFile: File) {
+        val modelData = ModelData(root = root.asExported())
+        outputFile.parentFile.mkdirs()
+        outputFile.writeText(modelData.toJson())
+    }
+}
+
+fun INode.asExported() : NodeData {
+    val idKey = NodeData.idPropertyKey
+    return NodeData(
+        id = getPropertyValue(idKey) ?: reference.serialize(),
+        concept = concept?.getUID(),
+        role = roleInParent,
+        properties = getPropertyRoles().associateWithNotNull { getPropertyValue(it) }.filterKeys { it != idKey },
+        references = getReferenceRoles().associateWithNotNull {
+            getReferenceTarget(it)?.getPropertyValue(idKey) ?: getReferenceTargetRef(it)?.serialize()
+        },
+        children = allChildren.map { it.asExported() }
+    )
+}

model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/ModelImporter.kt

@@ -0,0 +1,139 @@
+package org.modelix.model.sync
+
+import org.modelix.model.api.*
+import org.modelix.model.data.ModelData
+import org.modelix.model.data.NodeData
+import java.io.File
+
+class ModelImporter(private val root: INode) {
+
+    private val originalIdToExisting: MutableMap<String, INode> = mutableMapOf()
+    private val postponedReferences = ArrayList<() -> Unit>()
+    private val nodesToRemove = HashSet<INode>()
+
+    fun import(jsonFile: File) {
+        require(jsonFile.exists())
+        require(jsonFile.extension == "json")
+
+        val data = ModelData.fromJson(jsonFile.readText())
+        import(data)
+    }
+    
+    fun import(data: ModelData) {
+        originalIdToExisting.clear()
+        postponedReferences.clear()
+        nodesToRemove.clear()
+        buildExistingIndex(root)
+        data.root.originalId()?.let { originalIdToExisting[it] = root }
+        syncNode(root, data.root)
+        postponedReferences.forEach { it.invoke() }
+        nodesToRemove.forEach { it.remove() }
+    }
+
+    private fun syncNode(node: INode, data: NodeData) {
+        syncProperties(node, data)
+        syncChildren(node, data)
+        syncReferences(node, data)
+    }
+
+    private fun syncChildren(node: INode, data: NodeData) {
+        val allRoles = (data.children.map { it.role } + node.allChildren.map { it.roleInParent }).distinct()
+        for (role in allRoles) {
+            val expectedNodes = data.children.filter { it.role == role }
+            val existingNodes = node.getChildren(role).toList()
+
+            // optimization for when there is no change in the child list
+            // size check first to avoid querying the original ID
+            if (expectedNodes.size == existingNodes.size && expectedNodes.map { it.originalId() } == existingNodes.map { it.originalId() }) {
+                existingNodes.zip(expectedNodes).forEach { syncNode(it.first, it.second) }
+                continue
+            }
+
+            expectedNodes.forEachIndexed { index, expected ->
+                val nodeAtIndex = node.getChildren(role).toList().getOrNull(index)
+                val expectedId = checkNotNull(expected.originalId()) { "Specified node '$expected' has no id" }
+                val expectedConcept = expected.concept?.let { s -> ConceptReference(s) }
+                val childNode = if (nodeAtIndex?.originalId() != expectedId) {
+                    val existingNode = originalIdToExisting[expectedId]
+                    if (existingNode == null) {
+                        val newChild = node.addNewChild(role, index, expectedConcept)
+                        newChild.setPropertyValue(NodeData.idPropertyKey, expectedId)
+                        originalIdToExisting[expectedId] = newChild
+                        newChild
+                    } else {
+                        node.moveChild(role, index, existingNode)
+                        nodesToRemove.remove(existingNode)
+                        existingNode
+                    }
+                } else {
+                    nodeAtIndex
+                }
+                check(childNode.getConceptReference() == expectedConcept) { "Unexpected concept change" }
+
+                syncNode(childNode, expected)
+            }
+
+            nodesToRemove += node.getChildren(role).drop(expectedNodes.size)
+        }
+    }
+
+    private fun buildExistingIndex(root: INode) {
+        root.getDescendants(true).forEach {node ->
+            node.originalId()?.let { originalIdToExisting[it] = node }
+        }
+    }
+
+    private fun syncProperties(node: INode, nodeData: NodeData) {
+        if (node.getPropertyValue(NodeData.idPropertyKey) == null) {
+            node.setPropertyValue(NodeData.idPropertyKey, nodeData.originalId())
+        }
+
+        nodeData.properties.forEach {
+            if (node.getPropertyValue(it.key) != it.value) {
+                node.setPropertyValue(it.key, it.value)
+            }
+        }
+
+        val toBeRemoved = node.getPropertyRoles().toSet()
+            .subtract(nodeData.properties.keys)
+            .filter { it != NodeData.idPropertyKey }
+        toBeRemoved.forEach { node.setPropertyValue(it, null) }
+    }
+
+    private fun syncReferences(node: INode, nodeData: NodeData) {
+        nodeData.references.forEach {
+            val expectedTargetId = it.value
+            val actualTargetId = node.getReferenceTarget(it.key)?.originalId()
+            if (actualTargetId != expectedTargetId) {
+                val expectedTarget = originalIdToExisting[expectedTargetId]
+                if (expectedTarget == null) {
+                    postponedReferences += {
+                        val expectedRefTarget = originalIdToExisting[expectedTargetId]
+                        if (expectedRefTarget == null) {
+                            // The target node is not part of the model. Assuming it exists in some other model we can
+                            // store the reference and try to resolve it dynamically on access.
+                            node.setReferenceTarget(it.key, SerializedNodeReference(expectedTargetId))
+                        } else {
+                            node.setReferenceTarget(it.key, expectedRefTarget)
+                        }
+                    }
+                } else {
+                    node.setReferenceTarget(it.key, expectedTarget)
+                }
+            }
+        }
+        val toBeRemoved = node.getReferenceRoles().toSet() - nodeData.references.keys
+        toBeRemoved.forEach {
+            val nullReference: INodeReference? = null
+            node.setReferenceTarget(it, nullReference)
+        }
+    }
+}
+
+internal fun INode.originalId(): String? {
+    return this.getPropertyValue(NodeData.idPropertyKey)
+}
+
+internal fun NodeData.originalId(): String? {
+    return properties[NodeData.idPropertyKey] ?: id
+}