Embedder rework (#338)

* feat: embedder rework

* perf: deprecate Cohere and Python embedders

* chore: update submodules

* fix: search for embedders.json instead of reducer.pkl

* perf: update dump/load paths

* chore: update submodules

Author: andhreljaKern

controller/embedding/manager.py

@@ -230,7 +230,6 @@ def __recreate_or_extend_embedding(project_id: str, embedding_id: str) -> Embedd
 
         if (
             new_embedding_item.platform == enums.EmbeddingPlatform.OPENAI.value
-            or new_embedding_item.platform == enums.EmbeddingPlatform.COHERE.value
             or new_embedding_item.platform == enums.EmbeddingPlatform.AZURE.value
         ):
             agreement_item = agreement.get_by_xfkey(

controller/embedding/terms.py

@@ -11,16 +11,6 @@
         "terms": "Please note that by enabling this third-party API, you are stating that you accept its addition as a sub-processor under the terms of our Data Processing Agreement. Please be aware that the OpenAI API policies may conflict with your internal data and privacy policies. For more information please check: @@PLACEHOLDER@@. For questions you can contact us at [email protected].",
         "link": "https://openai.com/policies/api-data-usage-policies",
     },
-    EmbeddingPlatform.COHERE.value:{
-        "platform": EmbeddingPlatform.COHERE.value,
-        "terms": "Please note that by enabling this third-party API, you are stating that you accept its addition as a sub-processor under the terms of our Data Processing Agreement. Please be aware that the Cohere API policies may conflict with your internal data and privacy policies. For more information please check: @@PLACEHOLDER@@. For questions you can contact us at [email protected].",
-        "link": "https://cohere.com/terms-of-use",
-    },
-    EmbeddingPlatform.PYTHON.value: {
-        "platform": EmbeddingPlatform.PYTHON.value,
-        "terms": None,
-        "link": None,
-    },
     EmbeddingPlatform.AZURE.value: {
         "platform": EmbeddingPlatform.AZURE.value,
         "terms": "Please note that by enabling this third-party API, you are stating that you accept its addition as a sub-processor under the terms of our Data Processing Agreement. Please be aware that the Azure API policies may conflict with your internal data and privacy policies. For more information please check: @@PLACEHOLDER@@. For questions you can contact us at [email protected].",

controller/record/manager.py

@@ -270,7 +270,7 @@ def __check_and_prep_edit_records(
             continue
 
         emb_path = os.path.join(
-            "/inference", project_id, f"embedder-{str(embedding_item.id)}.pkl"
+            "/inference", project_id, f"embedder-{str(embedding_item.id)}.json"
         )
         if not os.path.exists(emb_path):
             errors_found.append(

controller/transfer/project_transfer_manager.py

@@ -339,10 +339,7 @@ def __transform_embedding_by_name(embedding_name: str):
         attribute_name = splitted_name[0]
         embedding_type = splitted_name[1]
         model = "-".join(splitted_name[2:])
-        if "bag-of-words" == model or "bag-of-characters" == model or "tf-idf" == model:
-            platform = enums.EmbeddingPlatform.PYTHON.value
-        else:
-            platform = enums.EmbeddingPlatform.HUGGINGFACE.value
+        platform = enums.EmbeddingPlatform.HUGGINGFACE.value
         name = f"{attribute_name}-{embedding_type}-{platform}-{model}"
         return platform, model, name