Release updates (#127)

* Dummy commit

* Adds enums support for export manager

* Adds running_id though pandas not db update

* Checks for IN_DELETEION projects on startup and finishes them up

* Submodules merge

---------

Co-authored-by: JWittmeyer <[email protected]>

Author: lumburovskalina

api/transfer.py

@@ -457,3 +457,4 @@ def __create_embeddings(
                 break
             time.sleep(1)
     return ctx_token
+

app.py

@@ -19,6 +19,7 @@
 
 from graphql_api import schema
 from controller.task_queue.task_queue import init_task_queue
+from controller.project.manager import check_in_deletion_projects
 
 
 logging.basicConfig(level=logging.DEBUG)
@@ -52,3 +53,4 @@
 app = Starlette(routes=routes, middleware=middleware)
 
 init_task_queue()
+check_in_deletion_projects()

controller/attribute/manager.py

@@ -141,7 +141,20 @@ def add_running_id(
 ) -> None:
     if attribute.get_by_name(project_id, attribute_name):
         raise ValueError(f"attribute with name {attribute_name} already exists")
+    general.commit()
 
+    # added threading for session management because otherwise this can sometimes create a deadlock
+    thread = daemon.prepare_thread(
+        __add_running_id, user_id, project_id, attribute_name, for_retokenization
+    )
+    thread.start()
+    thread.join()
+
+
+def __add_running_id(
+    user_id: str, project_id: str, attribute_name: str, for_retokenization: bool = True
+):
+    session_token = general.get_ctx_token()
     attribute.add_running_id(
         project_id, attribute_name, for_retokenization, with_commit=True
     )
@@ -156,6 +169,7 @@ def add_running_id(
                 "only_uploaded_attributes": False,
             },
         )
+    general.remove_and_refresh_session(session_token)
 
 
 def calculate_user_attribute_all_records(

controller/project/manager.py

@@ -19,6 +19,7 @@
     embedding,
     information_source,
     general,
+    organization,
 )
 from graphql_api.types import HuddleData, ProjectSize, GatesIntegrationData
 from util import daemon, notification
@@ -469,3 +470,22 @@ def __create_missing_information_source_pickles(
             time.sleep(1)
 
     return session_token
+
+
+def check_in_deletion_projects() -> None:
+    # this is only supposed to be called during startup of the application
+    daemon.run(__check_in_deletion_projects)
+
+
+def __check_in_deletion_projects() -> None:
+    # wait for startup to finish
+    time.sleep(2)
+    to_be_deleted = []
+    orgs = organization.get_all()
+    for org_item in orgs:
+        projects = project.get_all(str(org_item.id))
+        for project_item in projects:
+            if project_item.status == enums.ProjectStatus.IN_DELETION.value:
+                to_be_deleted.append(str(project_item.id))
+    for project_id in to_be_deleted:
+        delete_project(project_id)

controller/transfer/manager.py

@@ -67,7 +67,7 @@ def get_upload_credentials_and_id(
 
 def import_records_from_file(project_id: str, task: UploadTask) -> None:
     import_file(project_id, task)
-    check_and_add_running_id(project_id, str(task.user_id))
+    # adding of running_id now in pandas handling for performance reasons
     record_label_association.update_is_valid_manual_label_for_project(project_id)
     general.commit()
     check_and_update_null_labels(project_id, str(task.user_id))

controller/transfer/record_export_manager.py

@@ -229,11 +229,12 @@ def __get_record_data_query(project_id: str, row_options: Dict[str, Any]) -> str
 
 
 def __record_data_by_type(project_id: str, row_options: Dict[str, Any]) -> str:
-    if row_options.get("type") == "SLICE":
+    print(row_options, flush=True)
+    if row_options.get("type") == enums.RecordExportAmountTypes.SLICE.value:
         return ___record_data_by_slice(project_id, row_options.get("id"))
-    elif row_options.get("type") == "SESSION":
+    elif row_options.get("type") == enums.RecordExportAmountTypes.SESSION.value:
         return __record_data_by_session(project_id, row_options.get("id"))
-    elif row_options.get("type") == "ALL":
+    elif row_options.get("type") == enums.RecordExportAmountTypes.ALL.value:
         return __record_data_without_reducing(project_id)
     else:
         message = (

controller/transfer/record_transfer_manager.py

@@ -123,7 +123,7 @@ def import_file(project_id: str, upload_task: UploadTask) -> None:
     )
     record_category = category.infer_category(upload_task.file_name)
 
-    data = convert_to_record_dict(
+    data, added_col = convert_to_record_dict(
         file_type,
         tmp_file_name,
         upload_task.user_id,
@@ -134,7 +134,12 @@ def import_file(project_id: str, upload_task: UploadTask) -> None:
     import_records_and_rlas(
         project_id, upload_task.user_id, data, upload_task, record_category
     )
-
+    if added_col:
+        attribute_item = attribute.get_by_name(project_id, added_col)
+        attribute_item.relative_position = 0
+        attribute_item.is_primary_key = True
+        attribute_item.state = enums.AttributeState.AUTOMATICALLY_CREATED.value
+        general.commit()
     upload_task_manager.update_upload_task_to_finished(upload_task)
 
     user = user_manager.get_or_create_user(upload_task.user_id)

controller/transfer/util.py

@@ -3,13 +3,13 @@
 
 from submodules.model import enums
 from .checks import check_argument_allowed, run_checks, run_limit_checks
-from submodules.model.models import UploadTask, Attribute
+from submodules.model.models import UploadTask
 import pandas as pd
 from submodules.model.enums import NotificationType
+from submodules.model.business_objects import record
 import os
 import logging
 import traceback
-from submodules.model.business_objects import export
 from util import category
 from util import notification
 
@@ -45,7 +45,7 @@ def convert_to_record_dict(
     user_id: str,
     file_import_options: str,
     project_id: str,
-) -> List:
+) -> Tuple[List, str]:
     if not file_type:
         notification.create_notification(
             NotificationType.FILE_TYPE_NOT_GIVEN,
@@ -100,7 +100,28 @@ def convert_to_record_dict(
     run_limit_checks(df, project_id, user_id)
     run_checks(df, project_id, user_id)
     check_and_convert_category_for_unknown(df, project_id, user_id)
-    return df.to_dict("records")
+    added_col = add_running_id_if_not_present(df, project_id)
+    return df.to_dict("records"), added_col
+
+
+def add_running_id_if_not_present(df: pd.DataFrame, project_id: str) -> Optional[str]:
+    record_item = record.get_one(project_id)
+    if record_item:
+        # project already has records => no extensions of existing data
+        return
+    has_id_like = False
+    for key in df.columns:
+        if category.infer_category_enum(df, key) == enums.DataTypes.INTEGER.value:
+            has_id_like = True
+            break
+    if has_id_like:
+        return
+    col_name = "running_id"
+    while col_name in df.columns:
+        col_name += "_"
+    df[col_name] = df.index
+
+    return col_name
 
 
 def check_and_convert_category_for_unknown(
@@ -130,7 +151,7 @@ def string_to_import_option_dict(
         if len(tmp) == 2:
             parameter = tmp[0].strip()
             if not check_argument_allowed(parameter):
-                create_notification(
+                notification.create_notification(
                     NotificationType.UNKNOWN_PARAMETER,
                     user_id,
                     project_id,

submodules/model

@@ -2,4 +2,18 @@
 
 
 def run(target, *args, **kwargs):
-    threading.Thread(target=target, args=args, kwargs=kwargs, daemon=True,).start()
+    threading.Thread(
+        target=target,
+        args=args,
+        kwargs=kwargs,
+        daemon=True,
+    ).start()
+
+
+def prepare_thread(target, *args, **kwargs) -> threading.Thread:
+    return threading.Thread(
+        target=target,
+        args=args,
+        kwargs=kwargs,
+        daemon=True,
+    )