Merge pull request #1482 from modelix/lionweb2

LionWeb REST API in the Model Server

Author: slisson

bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/ModelExporter.kt

@@ -1,8 +1,12 @@
 package org.modelix.model.sync.bulk
 
+import org.modelix.kotlin.utils.DelicateModelixApi
 import org.modelix.model.api.INode
+import org.modelix.model.api.IReadableNode
+import org.modelix.model.api.getOriginalOrCurrentReference
+import org.modelix.model.api.getOriginalReference
+import org.modelix.model.api.meta.NullConcept
 import org.modelix.model.data.NodeData
-import org.modelix.model.data.associateWithNotNull
 
 /**
  * A ModelExporter exports a node and its subtree in bulk.
@@ -14,15 +18,21 @@ expect class ModelExporter(root: INode)
  * This function is recursively called on the node's children.
  */
 fun INode.asExported(): NodeData {
-    val idKey = NodeData.idPropertyKey
+    return asReadableNode().asExported()
+}
+
+fun IReadableNode.asExported(): NodeData {
+    @OptIn(DelicateModelixApi::class) // json file should contain role IDs
     return NodeData(
-        id = getPropertyValue(idKey) ?: reference.serialize(),
-        concept = getConceptReference()?.getUID(),
-        role = roleInParent,
-        properties = getPropertyRoles().associateWithNotNull { getPropertyValue(it) }.filterKeys { it != idKey },
-        references = getReferenceRoles().associateWithNotNull {
-            getReferenceTarget(it)?.getPropertyValue(idKey) ?: getReferenceTargetRef(it)?.serialize()
+        id = getOriginalOrCurrentReference(),
+        concept = getConceptReference().takeIf { it != NullConcept.getReference() }?.getUID(),
+        role = getContainmentLink().getIdOrNameOrNull(),
+        properties = getAllProperties()
+            .filterNot { it.first.matches(NodeData.ID_PROPERTY_REF) }
+            .associate { it.first.getIdOrName() to it.second },
+        references = getAllReferenceTargetRefs().associate {
+            it.first.getIdOrName() to (getReferenceTarget(it.first)?.getOriginalReference() ?: it.second.serialize())
         },
-        children = allChildren.map { it.asExported() },
+        children = getAllChildren().map { it.asExported() },
     )
 }

bulk-model-sync-mps/src/main/kotlin/org/modelix/mps/model/sync/bulk/MPSBulkSynchronizer.kt

@@ -25,7 +25,6 @@ import org.modelix.model.api.INode
 import org.modelix.model.api.TreePointer
 import org.modelix.model.api.getRootNode
 import org.modelix.model.client2.ModelClientV2
-import org.modelix.model.client2.lazyLoadVersion
 import org.modelix.model.data.ModelData
 import org.modelix.model.lazy.RepositoryId
 import org.modelix.model.mpsadapters.MPSArea

commitlint.config.js

@@ -8,6 +8,7 @@ module.exports = {
         "authorization",
         "bulk-model-sync",
         "bulk-model-sync-gradle",
+        "datastructures",
         "deps",
         "metamodel-export",
         "model-api-gen",

datastructures/build.gradle.kts

@@ -0,0 +1,35 @@
+plugins {
+    `maven-publish`
+    `modelix-kotlin-multiplatform`
+    alias(libs.plugins.kotlin.serialization)
+}
+
+kotlin {
+    sourceSets {
+        commonMain {
+            dependencies {
+                api(project(":kotlin-utils"))
+                api(project(":streams"))
+                implementation(libs.kotlin.serialization.core)
+                implementation(libs.kotlin.coroutines.core)
+                implementation(libs.kotlincrypto.sha2)
+            }
+        }
+        commonTest {
+            dependencies {
+                implementation(kotlin("test"))
+            }
+        }
+        jvmMain {
+            dependencies {
+                implementation(libs.trove4j)
+                implementation(libs.apache.commons.collections)
+                implementation(libs.kotlin.coroutines.core)
+            }
+        }
+        jsMain {
+            dependencies {
+            }
+        }
+    }
+}

datastructures/src/commonMain/kotlin/org/modelix/datastructures/IPersistentMap.kt

@@ -0,0 +1,39 @@
+package org.modelix.datastructures
+
+import org.modelix.datastructures.objects.IDataTypeConfiguration
+import org.modelix.datastructures.objects.IObjectData
+import org.modelix.datastructures.objects.Object
+import org.modelix.streams.IStream
+import org.modelix.streams.IStreamExecutorProvider
+
+/**
+ * Also works as a multimap. A multimap can store multiple entries with the same key. It depends on the configuration of
+ * the implementation whether it behaves as a multimap or not.
+ */
+interface IPersistentMap<K, V> : IStreamExecutorProvider {
+    fun asObject(): Object<*>
+    fun getKeyTypeConfig(): IDataTypeConfiguration<K>
+
+    fun putAll(entries: Iterable<Pair<K, V>>): IStream.One<IPersistentMap<K, V>>
+    fun removeAll(keys: Iterable<K>): IStream.One<IPersistentMap<K, V>>
+    fun removeAllEntries(entries: Iterable<Pair<K, V>>): IStream.One<IPersistentMap<K, V>>
+
+    fun getAllValues(keys: Iterable<K>): IStream.Many<V>
+    fun getAllValues(): IStream.Many<V>
+    fun getAll(keys: Iterable<K>): IStream.Many<Pair<K, V>>
+    fun getAll(): IStream.Many<Pair<K, V>>
+
+    fun get(key: K): IStream.ZeroOrOne<V> = getAll(listOf(key)).filter { it.first == key }.map { it.second }.firstOrEmpty()
+    fun put(key: K, value: V): IStream.One<IPersistentMap<K, V>> = putAll(listOf(key to value))
+    fun remove(key: K): IStream.One<IPersistentMap<K, V>> = removeAll(listOf(key))
+
+    fun getChanges(oldMap: IPersistentMap<K, V>, changesOnly: Boolean): IStream.Many<MapChangeEvent<K, V>>
+}
+
+interface IPersistentMapRootData<K, V> : IObjectData {
+    fun createMapInstance(self: Object<IPersistentMapRootData<K, V>>): IPersistentMap<K, V>
+}
+
+fun <K, V> Object<IPersistentMapRootData<K, V>>.createMapInstance(): IPersistentMap<K, V> {
+    return data.createMapInstance(this)
+}

datastructures/src/commonMain/kotlin/org/modelix/datastructures/MapChangeEvent.kt

@@ -0,0 +1,6 @@
+package org.modelix.datastructures
+
+sealed class MapChangeEvent<K, V>
+data class EntryAddedEvent<K, V>(val key: K, val value: V) : MapChangeEvent<K, V>()
+data class EntryRemovedEvent<K, V>(val key: K, val value: V) : MapChangeEvent<K, V>()
+data class EntryChangedEvent<K, V>(val key: K, val oldValue: V, val newValue: V) : MapChangeEvent<K, V>()

datastructures/src/commonMain/kotlin/org/modelix/datastructures/MapWithObjectReferenceValues.kt

@@ -0,0 +1,89 @@
+package org.modelix.datastructures
+
+import org.modelix.datastructures.objects.IDataTypeConfiguration
+import org.modelix.datastructures.objects.IObjectData
+import org.modelix.datastructures.objects.IObjectGraph
+import org.modelix.datastructures.objects.Object
+import org.modelix.datastructures.objects.ObjectReference
+import org.modelix.streams.IStream
+import org.modelix.streams.IStreamExecutorProvider
+
+class MapWithObjectReferenceValues<K, V : IObjectData>(
+    val graph: IObjectGraph,
+    val map: IPersistentMap<K, ObjectReference<V>>,
+) : IPersistentMap<K, V>, IStreamExecutorProvider by graph {
+
+    private fun IPersistentMap<K, ObjectReference<V>>.wrap() = MapWithObjectReferenceValues(graph, this)
+    private fun IStream.One<IPersistentMap<K, ObjectReference<V>>>.wrap() = map { it.wrap() }
+    private fun V.toRef() = graph.fromCreated(this)
+
+    override fun asObject(): Object<*> {
+        return map.asObject()
+    }
+
+    override fun getKeyTypeConfig(): IDataTypeConfiguration<K> {
+        return map.getKeyTypeConfig()
+    }
+
+    override fun putAll(entries: Iterable<Pair<K, V>>): IStream.One<MapWithObjectReferenceValues<K, V>> {
+        return map.putAll(entries.map { it.first to it.second.toRef() }).wrap()
+    }
+
+    override fun removeAll(keys: Iterable<K>): IStream.One<MapWithObjectReferenceValues<K, V>> {
+        return map.removeAll(keys).wrap()
+    }
+
+    override fun removeAllEntries(entries: Iterable<Pair<K, V>>): IStream.One<MapWithObjectReferenceValues<K, V>> {
+        return map.removeAllEntries(entries.map { it.first to it.second.toRef() }).wrap()
+    }
+
+    override fun getAllValues(keys: Iterable<K>): IStream.Many<V> {
+        return map.getAllValues(keys).flatMap { it.resolve() }.map { it.data }
+    }
+
+    override fun getAllValues(): IStream.Many<V> {
+        return map.getAllValues().flatMap { it.resolve() }.map { it.data }
+    }
+
+    override fun getAll(keys: Iterable<K>): IStream.Many<Pair<K, V>> {
+        return map.getAll(keys).flatMap { entry -> entry.second.resolve().map { value -> entry.first to value.data } }
+    }
+
+    override fun getAll(): IStream.Many<Pair<K, V>> {
+        return map.getAll().flatMap { entry -> entry.second.resolve().map { value -> entry.first to value.data } }
+    }
+
+    override fun put(key: K, value: V) = putAll(listOf(key to value))
+
+    override fun remove(key: K) = removeAll(listOf(key))
+
+    override fun getChanges(
+        oldMap: IPersistentMap<K, V>,
+        changesOnly: Boolean,
+    ): IStream.Many<MapChangeEvent<K, V>> {
+        oldMap as MapWithObjectReferenceValues<K, V>
+        return map.getChanges(oldMap.map, changesOnly).flatMap { event ->
+            when (event) {
+                is EntryAddedEvent<K, ObjectReference<V>> -> {
+                    event.value.resolveData().map { value ->
+                        EntryAddedEvent(event.key, value)
+                    }
+                }
+                is EntryChangedEvent<K, ObjectReference<V>> -> {
+                    event.oldValue.resolveData().zipWith(event.newValue.resolveData()) { oldValue, newValue ->
+                        EntryChangedEvent(event.key, oldValue, newValue)
+                    }
+                }
+                is EntryRemovedEvent<K, ObjectReference<V>> -> {
+                    event.value.resolveData().map { value ->
+                        EntryRemovedEvent(event.key, value)
+                    }
+                }
+            }
+        }
+    }
+}
+
+fun <K, V : IObjectData> IPersistentMap<K, ObjectReference<V>>.autoResolveValues(): IPersistentMap<K, V> {
+    return MapWithObjectReferenceValues(asObject().graph, this)
+}

datastructures/src/commonMain/kotlin/org/modelix/datastructures/btree/BTree.kt

@@ -0,0 +1,31 @@
+package org.modelix.datastructures.btree
+
+import org.modelix.datastructures.objects.IObjectGraph
+import org.modelix.kotlin.utils.DelicateModelixApi
+import org.modelix.streams.IStream
+
+data class BTree<K, V>(val root: BTreeNode<K, V>) {
+    constructor(config: BTreeConfig<K, V>) : this(BTreeNodeLeaf(config, emptyList()))
+
+    val graph: IObjectGraph get() = root.config.graph
+
+    fun validate() {
+        graph.getStreamExecutor().query {
+            root.validate(true)
+            @OptIn(DelicateModelixApi::class)
+            check(root.getEntries().toList().getSynchronous().map { it.key }.toSet().size == root.getEntries().map { it.key }.count().getSynchronous()) {
+                "duplicate entries: $root"
+            }
+            check(root.getEntries().map { it.key }.toList().getSynchronous().sortedWith(root.config.keyConfiguration) == root.getEntries().map { it.key }.toList().getSynchronous()) {
+                "not sorted: $this"
+            }
+            IStream.of(Unit)
+        }
+    }
+    fun put(key: K, value: V): BTree<K, V> = copy(root = root.put(key, value).createRoot())
+    fun get(key: K): V? = root.get(key)
+    fun getAll(keys: Iterable<K>): IStream.Many<Pair<K, V>> = root.getAll(keys)
+    fun remove(key: K): BTree<K, V> = copy(root = root.remove(key).createRoot())
+    fun remove(key: K, value: V): BTree<K, V> = copy(root = root.remove(key).createRoot())
+    fun getEntries(): IStream.Many<BTreeEntry<K, V>> = root.getEntries()
+}

datastructures/src/commonMain/kotlin/org/modelix/datastructures/btree/BTreeConfig.kt

@@ -0,0 +1,79 @@
+package org.modelix.datastructures.btree
+
+import org.modelix.datastructures.objects.Base64DataTypeConfiguration
+import org.modelix.datastructures.objects.IDataTypeConfiguration
+import org.modelix.datastructures.objects.IObjectGraph
+import org.modelix.datastructures.objects.LongDataTypeConfiguration
+import org.modelix.datastructures.objects.StringDataTypeConfiguration
+
+data class BTreeConfig<K, V>(
+    val minEntries: Int,
+    val minChildren: Int,
+    val multimap: Boolean,
+    val keyConfiguration: IDataTypeConfiguration<K>,
+    val valueConfiguration: IDataTypeConfiguration<V>,
+    val graph: IObjectGraph,
+) {
+    val maxEntries = 2 * minEntries
+    val maxChildren = 2 * minChildren
+    val nodeDeserializer = BTreeNode.Deserializer(this)
+    val entryComparatorForInsertion: Comparator<BTreeEntry<K, V>> = if (multimap) {
+        compareBy<BTreeEntry<K, V>, K>(keyConfiguration) { it.key }
+            .thenBy<BTreeEntry<K, V>, V>(valueConfiguration) { it.value }
+    } else {
+        compareBy<BTreeEntry<K, V>, K>(keyConfiguration) { it.key }
+    }
+
+    companion object {
+        fun builder() = BTreeConfigBuilder<Nothing, Nothing>()
+    }
+}
+
+class BTreeConfigBuilder<K, V> {
+    private var graph: IObjectGraph? = null
+    private var minEntries: Int = 8
+    private var minChildren: Int = 8
+    private var keyConfiguration: IDataTypeConfiguration<K>? = null
+    private var valueConfiguration: IDataTypeConfiguration<V>? = null
+    private var multimap: Boolean = false
+
+    fun minEntries(value: Int) = also { minEntries = value }
+    fun minChildren(value: Int) = also { minChildren = value }
+
+    fun maxEntries(value: Int) = also {
+        require(value % 2 == 0) { "Has to be a multiple of 2: $value" }
+        minEntries(value / 2)
+    }
+
+    fun maxChildren(value: Int) = also {
+        require(value % 2 == 0) { "Has to be a multiple of 2: $value" }
+        minChildren(value / 2)
+    }
+
+    fun graph(value: IObjectGraph) = also { graph = value }
+
+    fun <T> keyConfiguration(config: IDataTypeConfiguration<T>): BTreeConfigBuilder<T, V> {
+        return (this as BTreeConfigBuilder<T, V>).also { it.keyConfiguration = config }
+    }
+
+    fun <T> valueConfiguration(config: IDataTypeConfiguration<T>): BTreeConfigBuilder<K, T> {
+        return (this as BTreeConfigBuilder<K, T>).also { it.valueConfiguration = config }
+    }
+
+    fun multipleValuesPerKey() = also { multimap = true }
+    fun singleValuePerKey() = also { multimap = false }
+
+    fun stringKeys() = keyConfiguration(Base64DataTypeConfiguration(StringDataTypeConfiguration()))
+    fun stringValues() = valueConfiguration(Base64DataTypeConfiguration(StringDataTypeConfiguration()))
+    fun longKeys() = keyConfiguration(LongDataTypeConfiguration())
+    fun longValues() = valueConfiguration(LongDataTypeConfiguration())
+
+    fun build() = BTreeConfig(
+        graph = checkNotNull(graph) { "graph not specified" },
+        minEntries = minEntries,
+        minChildren = minChildren,
+        multimap = multimap,
+        keyConfiguration = checkNotNull(keyConfiguration) { "keyConfiguration not specified" },
+        valueConfiguration = checkNotNull(valueConfiguration) { "valueConfiguration not specified" },
+    )
+}

datastructures/src/commonMain/kotlin/org/modelix/datastructures/btree/BTreeEntry.kt

@@ -0,0 +1,7 @@
+package org.modelix.datastructures.btree
+
+class BTreeEntry<K, V>(val key: K, val value: V) {
+    override fun toString(): String {
+        return "$key -> $value"
+    }
+}