Merge pull request #568 from modelix/improve-model-client-memory-usage

perf(model-datastructure): use a memory efficient map for MapBasedStore

Author: Oleksandr Dzhychko

bulk-model-sync-lib/build.gradle.kts

@@ -21,12 +21,6 @@ kotlin {
             }
         }
 
-        val jvmMain by getting {
-            dependencies {
-                implementation(libs.trove4j)
-            }
-        }
-
         val commonTest by getting {
             dependencies {
                 implementation(project(":model-api"))

bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/ModelImporter.kt

@@ -17,6 +17,7 @@
 package org.modelix.model.sync.bulk
 
 import mu.KotlinLogging
+import org.modelix.kotlin.utils.createMemoryEfficientMap
 import org.modelix.model.api.ConceptReference
 import org.modelix.model.api.INode
 import org.modelix.model.api.INodeReference
@@ -256,8 +257,8 @@ class ModelImporter(
         }
     }
 
-    private fun buildExistingIndex(): MemoryEfficientMap<String, INodeReference> {
-        val localOriginalIdToExisting = MemoryEfficientMap<String, INodeReference>()
+    private fun buildExistingIndex(): MutableMap<String, INodeReference> {
+        val localOriginalIdToExisting = createMemoryEfficientMap<String, INodeReference>()
         root.getDescendants(true).forEach { node ->
             node.originalId()?.let { localOriginalIdToExisting[it] = node.reference }
         }

kotlin-utils/build.gradle.kts

@@ -30,6 +30,7 @@ kotlin {
         }
         val jvmMain by getting {
             dependencies {
+                implementation(libs.trove4j)
             }
         }
         val jvmTest by getting {

bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/MemoryEfficientMap.kt renamed to kotlin-utils/src/commonMain/kotlin/org/modelix/kotlin/utils/DataStructures.kt

@@ -14,21 +14,18 @@
  * limitations under the License.
  */
 
-package org.modelix.model.sync.bulk
+package org.modelix.kotlin.utils
 
 /**
- * Built-in maps like [HashMap] are not the most memory efficient way of map.
- * A common issue is that entry objects for every item in the table are created.
- * [MemoryEfficientMap] is an internal implementation that we can use
- * when the memory overhead becomes too big.
+ * Creates a mutable map with less memory overhead.
+ * This is an internal API.
  *
+ * Built-in maps like [HashMap] are not the not very memory efficient.
+ * A common issue is that entry objects for every item in the table are created.
  * Java implementation is optimized to not create entry objects by using a map implementation from another library.
  * The JS implementation is not optimized yet because we did not invest time in finding a suitable library.
  *
- * [MemoryEfficientMap] is an internal abstraction.
- * The API is therefore kept minimal
+ * We did not look into performance implications for storing and retrieving data.
+ * Therefore, the memory efficient maps are used sparingly for only the very big maps.
  */
-expect class MemoryEfficientMap<KeyT, ValueT>() {
-    operator fun set(key: KeyT, value: ValueT)
-    operator fun get(key: KeyT): ValueT?
-}
+expect fun <K, V> createMemoryEfficientMap(): MutableMap<K, V>

bulk-model-sync-lib/src/jvmMain/kotlin/org/modelix/model/sync/bulk/MemoryEfficientMap.kt renamed to kotlin-utils/src/jsMain/kotlin/org/modelix/kotlin/utils/DataStructures.js.kt

@@ -14,15 +14,6 @@
  * limitations under the License.
  */
 
-package org.modelix.model.sync.bulk
+package org.modelix.kotlin.utils
 
-import gnu.trove.map.TMap
-import gnu.trove.map.hash.THashMap
-
-actual class MemoryEfficientMap<KeyT, ValueT> {
-    private val map: TMap<KeyT, ValueT> = THashMap()
-
-    actual operator fun set(key: KeyT, value: ValueT) = map.set(key, value)
-
-    actual operator fun get(key: KeyT) = map[key]
-}
+actual fun <K, V> createMemoryEfficientMap(): MutableMap<K, V> = HashMap()

bulk-model-sync-lib/src/jsMain/kotlin/org/modelix/model/sync/bulk/MemoryEfficientMap.kt renamed to kotlin-utils/src/jvmMain/kotlin/org/modelix/kotlin/utils/DataStructures.jvm.kt

@@ -14,12 +14,8 @@
  * limitations under the License.
  */
 
-package org.modelix.model.sync.bulk
+package org.modelix.kotlin.utils
 
-actual class MemoryEfficientMap<KeyT, ValueT> {
-    private val map: MutableMap<KeyT, ValueT> = mutableMapOf()
+import gnu.trove.map.hash.THashMap
 
-    actual operator fun set(key: KeyT, value: ValueT) = map.set(key, value)
-
-    actual operator fun get(key: KeyT) = map[key]
-}
+actual fun <K, V> createMemoryEfficientMap(): MutableMap<K, V> = THashMap()

model-datastructure/src/commonMain/kotlin/org/modelix/model/persistent/MapBasedStore.kt

@@ -15,6 +15,7 @@
 
 package org.modelix.model.persistent
 
+import org.modelix.kotlin.utils.createMemoryEfficientMap
 import org.modelix.model.IKeyListener
 import org.modelix.model.IKeyValueStore
 import org.modelix.model.lazy.IBulkQuery
@@ -25,7 +26,7 @@ import org.modelix.model.lazy.NonBulkQuery
 open class MapBaseStore : MapBasedStore()
 
 open class MapBasedStore : IKeyValueStore {
-    private val map: MutableMap<String?, String?> = HashMap()
+    private val map: MutableMap<String?, String?> = createMemoryEfficientMap()
     override fun get(key: String): String? {
         return map[key]
     }