perf(bulk-model-sync): use a memory efficient map for critical indices

Use trove4js `THashMap` instead of Javas `HashMap` because it does not create entry objects for map items.

Author: Oleksandr Dzhychko

build.gradle.kts

@@ -116,7 +116,7 @@ subprojects {
         if (!name.lowercase().contains("test")) {
             this.kotlinOptions {
                 jvmTarget = "11"
-                freeCompilerArgs += listOf("-Xjvm-default=all-compatibility")
+                freeCompilerArgs += listOf("-Xjvm-default=all-compatibility", "-Xexpect-actual-classes")
                 apiVersion = kotlinApiVersion.version
             }
         }

bulk-model-sync-lib/build.gradle.kts

@@ -21,6 +21,12 @@ kotlin {
             }
         }
 
+        val jvmMain by getting {
+            dependencies {
+                implementation(libs.trove4j)
+            }
+        }
+
         val commonTest by getting {
             dependencies {
                 implementation(project(":model-api"))

bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/MemoryEfficientMap.kt

@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2024.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.modelix.model.sync.bulk
+
+/**
+ * Built-in maps like [HashMap] are not the most memory efficient way of map.
+ * A common issue is that entry objects for every item in the table are created.
+ * [MemoryEfficientMap] is an internal implementation that we can use
+ * when the memory overhead becomes too big.
+ *
+ * Java implementation is optimized to not create entry objects by using a map implementation from another library.
+ * The JS implementation is not optimized yet because we did not invest time in finding a suitable library.
+ *
+ * [MemoryEfficientMap] is an internal abstraction.
+ * The API is therefore kept minimal
+ */
+expect class MemoryEfficientMap<KeyT, ValueT>() {
+    operator fun set(key: KeyT, value: ValueT)
+    operator fun get(key: KeyT): ValueT?
+}

bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/ModelImporter.kt

@@ -48,8 +48,9 @@ class ModelImporter(
     private val continueOnError: Boolean,
     private val childFilter: (INode) -> Boolean = { true },
 ) {
-
-    private val originalIdToExisting: MutableMap<String, INode> = mutableMapOf()
+    // We have seen imports where the `originalIdToExisting` had a dozen ten million entries.
+    // Therefore, choose a map with is optimized for memory usage.
+    private var originalIdToExisting = MemoryEfficientMap<String, INode>()
     private val postponedReferences = mutableListOf<PostponedReference>()
     private val nodesToRemove = HashSet<INode>()
     private var numExpectedNodes = 0
@@ -99,7 +100,7 @@ class ModelImporter(
         INodeResolutionScope.runWithAdditionalScope(root.getArea()) {
             logImportSize(data.root, logger)
             logger.info { "Building indices for import..." }
-            originalIdToExisting.clear()
+            originalIdToExisting = MemoryEfficientMap()
             postponedReferences.clear()
             nodesToRemove.clear()
             numExpectedNodes = countExpectedNodes(data.root)
@@ -145,15 +146,25 @@ class ModelImporter(
         for (role in allRoles) {
             val expectedNodes = expectedParent.children.filter { it.role == role }
             val existingNodes = existingParent.getChildren(role).filter(childFilter).toList()
+            val allExpectedNodesDoNotExist by lazy {
+                expectedNodes.all { expectedNode ->
+                    val originalId = expectedNode.originalId()
+                    checkNotNull(originalId) { "Specified node '$expectedNode' has no ID." }
+                    originalIdToExisting[originalId] == null
+                }
+            }
 
             // optimization that uses the bulk operation .addNewChildren
-            if (existingNodes.isEmpty() && expectedNodes.all { originalIdToExisting[it.originalId()] == null }) {
-                existingParent.addNewChildren(role, -1, expectedNodes.map { it.concept?.let { ConceptReference(it) } }).zip(expectedNodes).forEach { (newChild, expected) ->
-                    val expectedId = checkNotNull(expected.originalId()) { "Specified node '$expected' has no id" }
-                    newChild.setPropertyValue(NodeData.idPropertyKey, expectedId)
-                    originalIdToExisting[expectedId] = newChild
-                    syncNode(newChild, expected, progressReporter)
-                }
+            if (existingNodes.isEmpty() && allExpectedNodesDoNotExist) {
+                existingParent.addNewChildren(role, -1, expectedNodes.map { it.concept?.let { ConceptReference(it) } })
+                    .zip(expectedNodes)
+                    .forEach { (newChild, expected) ->
+                        val expectedId = expected.originalId()
+                        checkNotNull(expectedId) { "Specified node '$expected' has no ID." }
+                        newChild.setPropertyValue(NodeData.idPropertyKey, expectedId)
+                        originalIdToExisting[expectedId] = newChild
+                        syncNode(newChild, expected, progressReporter)
+                    }
                 continue
             }
 

bulk-model-sync-lib/src/jsMain/kotlin/org/modelix/model/sync/bulk/MemoryEfficientMap.kt

@@ -0,0 +1,25 @@
+/*
+ * Copyright (c) 2024.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.modelix.model.sync.bulk
+
+actual class MemoryEfficientMap<KeyT, ValueT> {
+    private val map: MutableMap<KeyT, ValueT> = mutableMapOf()
+
+    actual operator fun set(key: KeyT, value: ValueT) = map.set(key, value)
+
+    actual operator fun get(key: KeyT) = map[key]
+}

bulk-model-sync-lib/src/jvmMain/kotlin/org/modelix/model/sync/bulk/MemoryEfficientMap.kt

@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2024.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.modelix.model.sync.bulk
+
+import gnu.trove.map.TMap
+import gnu.trove.map.hash.THashMap
+
+actual class MemoryEfficientMap<KeyT, ValueT> {
+    private val map: TMap<KeyT, ValueT> = THashMap()
+
+    actual operator fun set(key: KeyT, value: ValueT) = map.set(key, value)
+
+    actual operator fun get(key: KeyT) = map[key]
+}