release v1.1.0 (#7)

* Update LICENSE

* Bugfix: embedding creation fails for low number of records (#4)

* updates service endpoints in start script

* Fixes #2, no pca for attribute embeddings if low number of records

* Create dependabot.yml (#6)

* Notification for embedder warnings (#5)

* adds notification for embedder warnings

* adds embedders directory to gitignore

* prints last embedder warning in notification

* sends embedding warning as notification of type warning

* adds primary key or record_id to embedding warning

* fixes spelling

* updates embedders version number

Co-authored-by: JWittmeyer <[email protected]>

Co-authored-by: Felix Kirsch <[email protected]>

Author: JWittmeyer

.github/dependabot.yml

@@ -0,0 +1,14 @@
+version: 2
+updates:
+  # https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#package-ecosystem
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "daily"
+    # default is / which breaks drone
+    pull-request-branch-name:
+      separator: "-"
+    # not created automatically for version updates so only security ones are created
+    # https://docs.github.com/en/code-security/dependabot/dependabot-security-updates/configuring-dependabot-security-updates#overriding-the-default-behavior-with-a-configuration-file
+    open-pull-requests-limit: 0 
+    

.gitignore

@@ -129,4 +129,7 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
-.DS_Store
+.DS_Store
+
+# embedders package
+embedders/

LICENSE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright [yyyy] [name of copyright owner]
+   Copyright 2022 onetask.ai GmbH
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

controller.py

@@ -8,6 +8,7 @@
     record,
     tokenization,
     notification,
+    organization,
 )
 import torch
 import traceback
@@ -24,10 +25,9 @@
 from util import daemon, request_util
 from util.decorator import param_throttle
 from util.embedders import get_embedder
-from util.notification import send_project_update
+from util.notification import send_project_update, embedding_warning_templates
 import os
 import pandas as pd
-from submodules.model.business_objects import embedding, general, organization
 from submodules.s3 import controller as s3
 
 logging.basicConfig(level=logging.INFO)
@@ -133,7 +133,7 @@ def prepare_run_encoding(request: data_type.Request, embedding_type: str) -> int
                         request.project_id,
                         request.user_id,
                         message,
-                        "ERROR",
+                        enums.Notification.ERROR.value,
                         enums.NotificationType.EMBEDDING_CREATION_FAILED.value,
                         True,
                     )
@@ -142,11 +142,6 @@ def prepare_run_encoding(request: data_type.Request, embedding_type: str) -> int
                         f"notification_created:{request.user_id}",
                         True,
                     )
-                    embedding.update_embedding_state_failed(
-                        request.project_id,
-                        embedding_id,
-                        with_commit=True,
-                    )
                     doc_ock.post_embedding_failed(
                         request.user_id, request.config_string
                     )
@@ -175,7 +170,7 @@ def run_encoding(
             request.project_id,
             request.user_id,
             f"Initializing model {request.config_string}. This can take a few minutes.",
-            "INFO",
+            enums.Notification.INFO.value,
             enums.NotificationType.EMBEDDING_CREATION_STARTED.value,
             True,
         )
@@ -184,7 +179,9 @@ def run_encoding(
         )
         iso2_code = project.get_blank_tokenizer_from_project(request.project_id)
         try:
-            embedder = get_embedder(embedding_type, request.config_string, iso2_code)
+            embedder = get_embedder(
+                request.project_id, embedding_type, request.config_string, iso2_code
+            )
         except OSError:
             embedding.update_embedding_state_failed(
                 request.project_id,
@@ -201,7 +198,7 @@ def run_encoding(
                 request.project_id,
                 request.user_id,
                 message,
-                "ERROR",
+                enums.Notification.ERROR.value,
                 enums.NotificationType.EMBEDDING_CREATION_FAILED.value,
                 True,
             )
@@ -226,7 +223,7 @@ def run_encoding(
                 request.project_id,
                 request.user_id,
                 message,
-                "ERROR",
+                enums.Notification.ERROR.value,
                 enums.NotificationType.EMBEDDING_CREATION_FAILED.value,
                 True,
             )
@@ -240,7 +237,7 @@ def run_encoding(
             request.project_id,
             request.user_id,
             f"Could not load model {request.config_string}. Please contact the support.",
-            "ERROR",
+            enums.Notification.ERROR.value,
             enums.NotificationType.EMBEDDING_CREATION_FAILED.value,
             True,
         )
@@ -283,7 +280,7 @@ def run_encoding(
             request.project_id,
             request.user_id,
             f"Started encoding {attribute_name} using model {request.config_string}.",
-            "INFO",
+            enums.Notification.INFO.value,
             enums.NotificationType.EMBEDDING_CREATION_STARTED.value,
             True,
         )
@@ -324,6 +321,39 @@ def run_encoding(
                 initial_count,
             )
     except Exception:
+        for warning_type, idx_list in embedder.get_warnings().items():
+            # use last record with warning as example
+            example_record_id = record_ids[idx_list[-1]]
+
+            primary_keys = [
+                pk.name for pk in attribute.get_primary_keys(request.project_id)
+            ]
+            if primary_keys:
+                example_record_data = record.get(
+                    request.project_id, example_record_id
+                ).data
+                example_record_msg = "with primary key: " + ", ".join(
+                    [str(example_record_data[p_key]) for p_key in primary_keys]
+                )
+            else:
+                example_record_msg = " with record id: " + str(example_record_id)
+
+            warning_msg = embedding_warning_templates[warning_type].format(
+                record_number=len(idx_list), example_record_msg=example_record_msg
+            )
+
+            notification.create(
+                request.project_id,
+                request.user_id,
+                warning_msg,
+                enums.Notification.WARNING.value,
+                enums.NotificationType.EMBEDDING_CREATION_WARNING.value,
+                True,
+            )
+            send_project_update(
+                request.project_id, f"notification_created:{request.user_id}", True
+            )
+
         embedding.update_embedding_state_failed(
             request.project_id,
             embedding_id,
@@ -337,27 +367,51 @@ def run_encoding(
             request.project_id,
             request.user_id,
             "Error at runtime. Please contact support.",
-            "ERROR",
+            enums.Notification.ERROR.value,
             enums.NotificationType.EMBEDDING_CREATION_FAILED.value,
             True,
         )
         send_project_update(
             request.project_id, f"notification_created:{request.user_id}", True
         )
         print(traceback.format_exc(), flush=True)
-        embedding.update_embedding_state_failed(
-            request.project_id,
-            embedding_id,
-            with_commit=True,
-        )
-        send_project_update(
-            request.project_id,
-            f"embedding:{embedding_id}:state:{enums.EmbeddingState.FAILED.value}",
-        )
         doc_ock.post_embedding_failed(request.user_id, request.config_string)
         return 500
 
     if embedding.get(request.project_id, embedding_id):
+        for warning_type, idx_list in embedder.get_warnings().items():
+            # use last record with warning as example
+            example_record_id = record_ids[idx_list[-1]]
+
+            primary_keys = [
+                pk.name for pk in attribute.get_primary_keys(request.project_id)
+            ]
+            if primary_keys:
+                example_record_data = record.get(
+                    request.project_id, example_record_id
+                ).data
+                example_record_msg = "with primary key: " + ", ".join(
+                    [str(example_record_data[p_key]) for p_key in primary_keys]
+                )
+            else:
+                example_record_msg = " with record id: " + str(example_record_id)
+
+            warning_msg = embedding_warning_templates[warning_type].format(
+                record_number=len(idx_list), example_record_msg=example_record_msg
+            )
+
+            notification.create(
+                request.project_id,
+                request.user_id,
+                warning_msg,
+                enums.Notification.WARNING.value,
+                enums.NotificationType.EMBEDDING_CREATION_WARNING.value,
+                True,
+            )
+            send_project_update(
+                request.project_id, f"notification_created:{request.user_id}", True
+            )
+
         if embedding_type == "classification":
             request_util.post_embedding_to_neural_search(
                 request.project_id, embedding_id
@@ -376,7 +430,7 @@ def run_encoding(
             request.project_id,
             request.user_id,
             f"Finished encoding {attribute_name} using model {request.config_string}.",
-            "SUCCESS",
+            enums.Notification.SUCCESS.value,
             enums.NotificationType.EMBEDDING_CREATION_DONE.value,
             True,
         )

requirements.txt

@@ -7,7 +7,7 @@ certifi==2021.10.8
 charset-normalizer==2.0.12
 click==8.0.4
 cymem==2.0.6
-embedders==0.0.14
+embedders==0.0.15
 fastapi==0.78.0
 greenlet==1.1.2
 h11==0.13.0

start

@@ -19,9 +19,9 @@ docker run -d --rm \
 -e S3_ACCESS_KEY=onetask \
 -e S3_SECRET_KEY=r6ywtR33!DMlaL*SUUdy \
 -e POSTGRES=postgresql://postgres:onetask@graphql-postgres:5432 \
--e DOC_OCK=http://doc-ock:80 \
--e WS_NOTIFY_ENDPOINT="http://ws-notify:8080" \
--e NEURAL_SEARCH=http://neural-search:80 \
+-e DOC_OCK=http://refinery-doc-ock:80 \
+-e WS_NOTIFY_ENDPOINT="http://refinery-websocket:8080" \
+-e NEURAL_SEARCH=http://refinery-neural-search:80 \
 --mount type=bind,source="$(pwd)"/,target=/app \
 -v /var/run/docker.sock:/var/run/docker.sock \
 --network dev-setup_default \

submodules/model

@@ -13,33 +13,32 @@
 from embedders.extraction.reduce import PCATokenReducer
 from embedders import Transformer
 
+from submodules.model.business_objects import record
 
 def get_embedder(
-    embedding_type: str, config_string: str, language_code: str
+    project_id: str, embedding_type: str, config_string: str, language_code: str,
 ) -> Transformer:
     if embedding_type == "classification":
         batch_size = 128
         n_components = 64
         if config_string == "bag-of-characters":
             return BagOfCharsSentenceEmbedder(batch_size=batch_size)
-        if config_string == "bag-of-words":
-            return PCASentenceReducer(
-                BagOfWordsSentenceEmbedder(batch_size=batch_size),
-                n_components=n_components,
-            )
-        if config_string == "tf-idf":
-            return PCASentenceReducer(
-                TfidfSentenceEmbedder(batch_size=batch_size), n_components=n_components
-            )
-        if config_string == "word2vec":
+        elif config_string == "bag-of-words":
+            embedder = BagOfWordsSentenceEmbedder(batch_size=batch_size)
+        elif config_string == "tf-idf":
+            embedder = TfidfSentenceEmbedder(batch_size=batch_size),
+        elif config_string == "word2vec":
             return None
         else:
-            return PCASentenceReducer(
-                TransformerSentenceEmbedder(
+            embedder = TransformerSentenceEmbedder(
                     config_string=config_string, batch_size=batch_size
-                ),
-                n_components=n_components,
-            )
+                )
+
+        if record.count(project_id) < n_components:
+            return embedder
+        else:
+            return PCASentenceReducer(embedder, n_components = n_components)
+
     else:  # extraction
         batch_size = 32
         n_components = 16

submodules/s3

@@ -1,8 +1,23 @@
 import requests
 import os
 
+from embedders.enums import WarningType
+
 from submodules.model.business_objects import project
 
+embedding_warning_templates = {
+    WarningType.DOCUMENT_IS_SPLITTED.value: (
+        "For {record_number} records, the text length exceeds the model's max input"
+        " length. For these records, the texts are splitted and the parts are processed"
+        " individually. For example, record {example_record_msg}."
+    ),
+    WarningType.TOKEN_MISMATCHING.value: (
+        "For {record_number} records, the number of embeddings does not match the "
+        "number of spacy tokens. Please contact support. For example, record "
+        "{example_record_msg}."
+    ),
+}
+
 
 def send_project_update(project_id: str, message: str, is_global: bool = False) -> None:
     endpoint = os.getenv("WS_NOTIFY_ENDPOINT")