Release preparations (#154)

* Adds newer bricks naming sheme for string detection

* Adds jsonify of list attributes

* Error message for recreating azure and openai embeddings

* Submodules merged

---------

Co-authored-by: Lina <[email protected]>

Author: JWittmeyer

controller/embedding/manager.py

@@ -1,5 +1,6 @@
 import time
 from typing import Any, Dict, List, Optional
+from exceptions.exceptions import ApiTokenImportError
 
 from submodules.model import enums
 from submodules.model.models import Embedding
@@ -108,7 +109,7 @@ def get_embedding_name(
 
 
 def recreate_embeddings(
-    project_id: str, embedding_ids: Optional[List[str]] = None
+    project_id: str, embedding_ids: Optional[List[str]] = None, user_id: str = None
 ) -> None:
     if not embedding_ids:
         embeddings = embedding.get_all_embeddings_by_project_id(project_id)
@@ -148,29 +149,37 @@ def recreate_embeddings(
                     break
                 else:
                     time.sleep(1)
+        except ApiTokenImportError as e:
+            notification.create_notification(enums.NotificationType.RECREATION_OF_EMBEDDINGS_ERROR, user_id, project_id)
+            __handle_failed_embedding(project_id,embedding_id, new_id,e)
+            
         except Exception as e:
-            print(
-                f"Error while recreating embedding for {project_id} with id {embedding_id} - {e}",
-                flush=True,
-            )
-            notification.send_organization_update(
-                project_id,
-                f"embedding:{embedding_id}:state:{enums.EmbeddingState.FAILED.value}",
-            )
-            old_embedding_item = embedding.get(project_id, embedding_id)
-            if old_embedding_item:
-                old_embedding_item.state = enums.EmbeddingState.FAILED.value
-
-            if new_id:
-                new_embedding_item = embedding.get(project_id, new_id)
-                if new_embedding_item:
-                    new_embedding_item.state = enums.EmbeddingState.FAILED.value
-            general.commit()
+            __handle_failed_embedding(project_id,embedding_id, new_id,e)
 
         notification.send_organization_update(
             project_id=project_id, message="embedding:finished:all"
         )
 
+def __handle_failed_embedding(project_id: str, embedding_id: str,new_id: str, e: Exception) -> None:
+        print(
+            f"Error while recreating embedding for {project_id} with id {embedding_id} - {e}",
+            flush=True,
+        )
+        
+        notification.send_organization_update(
+            project_id,
+            f"embedding:{embedding_id}:state:{enums.EmbeddingState.FAILED.value}",
+        )
+        old_embedding_item = embedding.get(project_id, embedding_id)
+        if old_embedding_item:
+            old_embedding_item.state = enums.EmbeddingState.FAILED.value
+
+        if new_id:
+            new_embedding_item = embedding.get(project_id, new_id)
+            if new_embedding_item:
+                new_embedding_item.state = enums.EmbeddingState.FAILED.value
+        general.commit()
+
 
 def __recreate_embedding(project_id: str, embedding_id: str) -> Embedding:
     old_embedding_item = embedding.get(project_id, embedding_id)
@@ -204,7 +213,7 @@ def __recreate_embedding(project_id: str, embedding_id: str) -> Embedding:
         if not agreement_item:
             new_embedding_item.state = enums.EmbeddingState.FAILED.value
             general.commit()
-            raise Exception(
+            raise ApiTokenImportError(
                 f"No agreement found for embedding {new_embedding_item.name}"
             )
         agreement_item.xfkey = new_embedding_item.id

controller/notification/notification_data.py

@@ -361,6 +361,13 @@
         "page": enums.Pages.SETTINGS.value,
         "docs": enums.DOCS.CREATING_PROJECTS.value,
     },
+    enums.NotificationType.RECREATION_OF_EMBEDDINGS_ERROR.value: {
+        "message_template": "Recreation of embeddings failed. Third party embeddings must be recreated manually.",
+        "title": "Embedding creation",
+        "level": enums.Notification.ERROR.value,
+        "page": enums.Pages.SETTINGS.value,
+        "docs": enums.DOCS.CREATE_EMBEDDINGS.value,
+    },
 }
 
 

controller/payload/payload_scheduler.py

@@ -149,7 +149,6 @@ def prepare_input_data_for_payload(
             information_source_item.type
             == enums.InformationSourceType.ACTIVE_LEARNING.value
         ):
-
             # for active learning, we can not evaluate on all records that are used for training
             # as otherwise, we would retrieve a false understanding of the accuracy!
             add_information_source_statistics_exclusion(
@@ -211,7 +210,6 @@ def execution_pipeline(
         add_file_name: str,
         input_data: Dict[str, Any],
     ) -> None:
-
         if (
             information_source_item.type
             == enums.InformationSourceType.LABELING_FUNCTION.value
@@ -469,7 +467,6 @@ def read_container_logs_thread(
     payload_id: str,
     docker_container: Any,
 ):
-
     ctx_token = general.get_ctx_token()
     # needs to be refetched since it is not thread safe
     information_source_payload = information_source.get_payload(project_id, payload_id)
@@ -782,6 +779,14 @@ def __get_embedding_id_from_function(
         re.IGNORECASE,
     )
     if not embedding_name:
+        # newer bricks modules
+        embedding_name = re.search(
+            r'EMBEDDING: str = "([\w\W]+?)"',
+            source_item.source_code,
+            re.IGNORECASE,
+        )
+    if not embedding_name:
+        # older bricks modules
         embedding_name = re.search(
             r'YOUR_EMBEDDING: str = "([\w\W]+?)"',
             source_item.source_code,
@@ -863,7 +868,6 @@ def prepare_sample_records_doc_bin(
 def run_labeling_function_exec_env(
     project_id: str, information_source_id: str, prefixed_doc_bin: str
 ) -> Tuple[List[str], List[List[str]], bool]:
-
     information_source_item = information_source.get(project_id, information_source_id)
 
     prefixed_function_name = f"{information_source_id}_fn"

controller/transfer/project_transfer_manager.py

@@ -397,7 +397,6 @@ def __transform_embedding_by_name(embedding_name: str):
                     "id",
                 )
             ] = embedding_object.id
-
         if data.get(
             "embedding_tensors_data",
         ):
@@ -904,14 +903,14 @@ def __replace_embedding_name(
 
     general.commit()
     daemon.run(
-        __post_processing_import_threaded, project_id, task_id, embedding_ids, data
+        __post_processing_import_threaded, project_id, task_id, embedding_ids, data, import_user_id
     )
     send_progress_update(project_id, task_id, 100)
     logger.info(f"Finished import of project {project_id}")
 
 
 def __post_processing_import_threaded(
-    project_id: str, task_id: str, embedding_ids: List[str], data: Dict[str, Any]
+    project_id: str, task_id: str, embedding_ids: List[str], data: Dict[str, Any], user_id: str
 ) -> None:
     time.sleep(5)
     while True:
@@ -926,7 +925,7 @@ def __post_processing_import_threaded(
     if not data.get(
         "embedding_tensors_data",
     ):
-        embedding_manager.recreate_embeddings(project_id)
+        embedding_manager.recreate_embeddings(project_id,user_id=user_id)
     else:
         for old_id in embedding_ids:
             embedding_manager.request_tensor_upload(

controller/transfer/util.py

@@ -103,7 +103,7 @@ def convert_to_record_dict(
     run_checks(df, project_id, user_id)
     check_and_convert_category_for_unknown(df, project_id, user_id)
 
-    df = covert_nested_attributes_to_text(df)
+    covert_nested_attributes_to_text(df)
     added_col = add_running_id_if_not_present(df, project_id)
     return df.to_dict("records"), added_col
 
@@ -148,14 +148,13 @@ def check_and_convert_category_for_unknown(
 def covert_nested_attributes_to_text(df: pd.DataFrame) -> pd.DataFrame:
     for key in df.columns:
         sample = pick_sample(df, key)
-        if check_sample_has_dict_values(sample):
+        if check_sample_has_dict_or_list_values(sample):
             df[key] = df[key].apply(lambda x: json.dumps(x))
-    return df
 
 
-def check_sample_has_dict_values(sample: List[Any]) -> bool:
+def check_sample_has_dict_or_list_values(sample: List[Any]) -> bool:
     for value in sample:
-        if isinstance(value, dict):
+        if isinstance(value, dict) or isinstance(value, list):
             return True
     return False
 

exceptions/exceptions.py

@@ -23,3 +23,6 @@ class NotAllowedInDemoError(Exception):
 
 class BadPasswordError(Exception):
     pass
+
+class ApiTokenImportError(Exception):
+    pass