perf: Make import quicker by caching imported translations, and using them when checking for conflicts instead of using ImportDataManager, which queries the database

Author: gabrielshanahan

backend/data/src/main/kotlin/io/tolgee/service/dataImport/CoreImportFilesProcessor.kt

@@ -65,6 +65,9 @@ class CoreImportFilesProcessor(
   val errors = mutableListOf<ErrorResponseBody>()
   val warnings = mutableListOf<ErrorResponseBody>()
 
+  private val importedTranslations =
+    mutableMapOf<ImportLanguage, MutableMap<ImportKey, MutableList<ImportTranslation>>>()
+
   fun processFiles(files: Collection<ImportFileDto>?) {
     errors.addAll(processFilesRecursive(files))
     renderPossibleNamespacesWarning()
@@ -330,9 +333,17 @@ class CoreImportFilesProcessor(
     }
 
     translationsByKeys.forEach { (key, translations) ->
-      translations.forEach {
-        it.key = key
-        processTranslation(it)
+      translations.forEach { translation ->
+        importedTranslations.putIfAbsent(translation.language, mutableMapOf())
+        importedTranslations.getValue(translation.language).putIfAbsent(key, mutableListOf())
+        importedTranslations.getValue(translation.language).getValue(key).add(translation)
+      }
+    }
+
+    translationsByKeys.forEach { (key, translations) ->
+      translations.forEach { translation ->
+        translation.key = key
+        processTranslation(translation)
       }
     }
 
@@ -371,9 +382,10 @@ class CoreImportFilesProcessor(
     var isCollision = false
     val issues =
       mutableListOf<Pair<FileIssueType, Map<FileIssueParamType, String>>>()
-    val storedTranslations =
-      importDataManager
-        .getStoredTranslations(newTranslation.key, newTranslation.language)
+    val storedTranslations = importedTranslations[newTranslation.language]
+      ?.get(newTranslation.key)
+      ?: emptyList()
+
     if (storedTranslations.isNotEmpty()) {
       isCollision = true
       storedTranslations.forEach { collision ->

backend/data/src/main/kotlin/io/tolgee/service/dataImport/ImportDataManager.kt

@@ -108,24 +108,6 @@ class ImportDataManager(
     namespaceService.getAllInProject(import.project.id).map { it.name to it }.toMap(mutableMapOf())
   }
 
-  /**
-   * Returns list of translations provided for a language and a key.
-   * It returns collection since translations could collide, when a user uploads a file with different values
-   * for a key
-   */
-  fun getStoredTranslations(
-    key: ImportKey,
-    language: ImportLanguage,
-  ): MutableList<ImportTranslation> {
-    this.populateStoredTranslations(language)
-    val languageData = this.storedTranslations[language]!!
-
-    return languageData[key] ?: let {
-      languageData[key] = mutableListOf()
-      languageData[key]!!
-    }
-  }
-
   fun getStoredTranslations(language: ImportLanguage): List<ImportTranslation> {
     return this.populateStoredTranslations(language).flatMap { it.value }
   }