From 43d447956807f11c9d93aef9766e76a16e24f408 Mon Sep 17 00:00:00 2001 From: JWittmeyer Date: Mon, 7 Jul 2025 14:16:00 +0200 Subject: [PATCH 01/11] Change num workers to 25 for azure foundry and others to 50 --- controller/attribute/util.py | 9 ++++++++- submodules/model | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/controller/attribute/util.py b/controller/attribute/util.py index b715c9d4..6a88933e 100644 --- a/controller/attribute/util.py +++ b/controller/attribute/util.py @@ -250,7 +250,6 @@ def prepare_llm_response_code( llm_playground_config: Union[Dict[str, Any], None] = None, llm_ac_cache_access_link: Union[str, None] = None, llm_ac_cache_file_upload_link: Union[str, None] = None, - num_workers: int = 100, max_api_call_retries: int = 5, retry_sleep_seconds: int = 5, ) -> str: @@ -294,6 +293,14 @@ async def ac(record): ) validate_llm_config(llm_config=llm_config) + num_workers = 50 + if ( + llm_config is not None + and enums.LLMProvider.from_string(llm_config.get("llmIdentifier", "Open ai")) + == enums.LLMProvider.AZURE_FOUNDRY + ): + num_workers = 25 + try: llm_config_mapping = { "@@API_KEY@@": llm_config["apiKey"], diff --git a/submodules/model b/submodules/model index 19c0a4d2..6ef421f4 160000 --- a/submodules/model +++ b/submodules/model @@ -1 +1 @@ -Subproject commit 19c0a4d25233fa0a7d5c4ee5377954c0594d2750 +Subproject commit 6ef421f4c87bcf8679435be4f76e11260c632e17 From 0294dedbb421340d6e65a8436b506df82762b120 Mon Sep 17 00:00:00 2001 From: Lina Date: Mon, 7 Jul 2025 14:22:17 +0200 Subject: [PATCH 02/11] The flag for projects from integrations is missing --- controller/project/manager.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/controller/project/manager.py b/controller/project/manager.py index 78c91c1f..d5f688a1 100644 --- a/controller/project/manager.py +++ b/controller/project/manager.py @@ -155,6 +155,11 @@ def get_all_projects_by_user(organization_id) -> List[Project]: else: p["num_data_scale_uploaded"] = record.get_count_scale_uploaded(p["id"]) + p["is_integration_project"] = any( + p["name"].startswith(f"[{integration_type.value}]") + for integration_type in enums.CognitionIntegrationType + ) + del p["created_by"] return project_dicts From 6ac49253df477f331bc0b6848a3612e54b4f3815 Mon Sep 17 00:00:00 2001 From: Lina Date: Mon, 7 Jul 2025 14:51:07 +0200 Subject: [PATCH 03/11] Count not displayed on embeddings table --- fast_api/routes/embedding.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/fast_api/routes/embedding.py b/fast_api/routes/embedding.py index d482adec..6e51dbe6 100644 --- a/fast_api/routes/embedding.py +++ b/fast_api/routes/embedding.py @@ -16,7 +16,7 @@ from controller.auth import manager as auth_manager from controller.embedding.connector import collection_on_qdrant from submodules.model.enums import TaskType -from submodules.model.business_objects import embedding +from submodules.model.business_objects import embedding as embedding_bo from submodules.model.cognition_objects import environment_variable as env_var_db_bo from submodules.model.util import sql_alchemy_to_dict from util import notification, spacy_util @@ -51,13 +51,16 @@ def language_models(request: Request) -> List: dependencies=[Depends(auth_manager.check_project_access_dep)], ) def get_embeddings(project_id: str) -> List: - embeddings_extended = embedding.get_all_embeddings_by_project_id_extended( + embeddings_extended = embedding_bo.get_all_embeddings_by_project_id_extended( project_id ) data = [ { **sql_alchemy_to_dict(embedding), "on_qdrant": collection_on_qdrant(project_id, embedding["id"]), + "count": sql_alchemy_to_dict( + embedding_bo.get_tensor_count(embedding["id"]) + ), } for embedding in embeddings_extended ] From f4a1046bc86e97416acebb2d1be5a6848920e180 Mon Sep 17 00:00:00 2001 From: andhreljaKern Date: Mon, 7 Jul 2025 20:49:42 +0200 Subject: [PATCH 04/11] fix: org buttons failure --- controller/misc/manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controller/misc/manager.py b/controller/misc/manager.py index a6cb8227..29be80bd 100644 --- a/controller/misc/manager.py +++ b/controller/misc/manager.py @@ -93,7 +93,7 @@ def finalize_customer_buttons( for e in buttons: e[key_name] = name_lookup[str(e[key])] e[key_name] = ( - (e[key_name]["first"] + " " + e[key_name]["last"]) + (e[key_name]["first"] or "" + " " + e[key_name]["last"] or "") if e[key_name] else "Unknown" ) From c216941f0bfd44f1547180b87efb183652be37fd Mon Sep 17 00:00:00 2001 From: andhreljaKern Date: Mon, 7 Jul 2025 20:50:04 +0200 Subject: [PATCH 05/11] chore: update submodules --- submodules/model | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submodules/model b/submodules/model index 6ef421f4..5ec9519b 160000 --- a/submodules/model +++ b/submodules/model @@ -1 +1 @@ -Subproject commit 6ef421f4c87bcf8679435be4f76e11260c632e17 +Subproject commit 5ec9519b7cf2bc033017dc3c25fa5bfe3eefafdd From 63e7d2c70eaa1dbba6520fb4f3872b6a16d09cfc Mon Sep 17 00:00:00 2001 From: Lina Date: Tue, 8 Jul 2025 15:35:55 +0200 Subject: [PATCH 06/11] Assign role when inviting user --- controller/auth/manager.py | 8 +++++++- fast_api/models.py | 1 + fast_api/routes/misc.py | 4 +++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/controller/auth/manager.py b/controller/auth/manager.py index e7a1c31b..79d9ba97 100644 --- a/controller/auth/manager.py +++ b/controller/auth/manager.py @@ -174,7 +174,10 @@ def check_is_full_admin(request: Any) -> bool: def invite_users( - emails: List[str], organization_name: str, provider: Optional[str] = None + emails: List[str], + organization_name: str, + user_role: str, + provider: Optional[str] = None, ): user_ids = [] for email in emails: @@ -186,6 +189,9 @@ def invite_users( # Assign the account to the organization user_manager.update_organization_of_user(organization_name, email) + # Assign the user role + user_manager.update_user_role(user["id"], user_role) + # Get the recovery link for the email recovery_link = kratos.get_recovery_link(user["id"]) if not recovery_link: diff --git a/fast_api/models.py b/fast_api/models.py index bd667a9b..ab44f950 100644 --- a/fast_api/models.py +++ b/fast_api/models.py @@ -502,6 +502,7 @@ class InviteUsersBody(BaseModel): emails: List[StrictStr] organization_name: StrictStr provider: Optional[StrictStr] = None + user_role: StrictStr class CheckInviteUsersBody(BaseModel): diff --git a/fast_api/routes/misc.py b/fast_api/routes/misc.py index 9ad178c8..aae1f4a9 100644 --- a/fast_api/routes/misc.py +++ b/fast_api/routes/misc.py @@ -293,7 +293,9 @@ def get_is_full_admin(request: Request) -> Dict: def invite_users(request: Request, body: InviteUsersBody = Body(...)): if not auth.check_is_full_admin(request): raise AuthManagerError("Full admin access required") - data = auth.invite_users(body.emails, body.organization_name, body.provider) + data = auth.invite_users( + body.emails, body.organization_name, body.user_role, body.provider + ) return pack_json_result(data) From 7bfde341effb6c23f347a6829345a328cc35ec21 Mon Sep 17 00:00:00 2001 From: Lina Date: Tue, 8 Jul 2025 16:32:44 +0200 Subject: [PATCH 07/11] Message for invalid emails --- controller/auth/manager.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/controller/auth/manager.py b/controller/auth/manager.py index 79d9ba97..f3bcca27 100644 --- a/controller/auth/manager.py +++ b/controller/auth/manager.py @@ -19,7 +19,7 @@ DEV_USER_ID = "741df1c2-a531-43b6-b259-df23bc78e9a2" -EMAIL_RE = re.compile(r"[\w\.-]+@[\w\.-]+\.\w+") +EMAIL_RE = re.compile(r"^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}$") def get_organization_id_by_info(info) -> Organization: @@ -203,13 +203,18 @@ def invite_users( def check_valid_emails(emails: List[str]): - valid_emails = [ - email - for email in emails - if is_valid_email(email) and not kratos.check_user_exists(email) - ] + message = "" + valid_emails = [] + for email in emails: + if not is_valid_email(email): + message += f"{email} is not a valid email address." + elif kratos.check_user_exists(email): + message += f"{email} already exists." + else: + valid_emails.append(email) + all_valid = len(valid_emails) == len(emails) - return {"valid_emails": valid_emails, "all_valid": all_valid} + return {"valid_emails": valid_emails, "all_valid": all_valid, "message": message} def is_valid_email(email: str) -> bool: From 50519d220acf09b979f87e06c27a89b408df8411 Mon Sep 17 00:00:00 2001 From: JWittmeyer Date: Tue, 8 Jul 2025 17:05:37 +0200 Subject: [PATCH 08/11] Submodule update --- submodules/model | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submodules/model b/submodules/model index 5ec9519b..61ee231c 160000 --- a/submodules/model +++ b/submodules/model @@ -1 +1 @@ -Subproject commit 5ec9519b7cf2bc033017dc3c25fa5bfe3eefafdd +Subproject commit 61ee231c61dc3509b79b6fc738f5871e77911ecb From 69ecb3a620f7b36e183785e883af0aad2f9d5db4 Mon Sep 17 00:00:00 2001 From: JWittmeyer Date: Wed, 9 Jul 2025 11:19:24 +0200 Subject: [PATCH 09/11] change logic --- controller/project/manager.py | 12 +++++++----- submodules/model | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/controller/project/manager.py b/controller/project/manager.py index d5f688a1..716d129b 100644 --- a/controller/project/manager.py +++ b/controller/project/manager.py @@ -23,6 +23,7 @@ from controller.task_master import manager as task_master_manager from submodules.model.enums import TaskType, RecordTokenizationScope from submodules.model.business_objects import util as db_util +from submodules.model.cognition_objects import integration as integration_db_co from submodules.model.integration_objects.helper import ( REFINERY_ATTRIBUTE_ACCESS_GROUPS, REFINERY_ATTRIBUTE_ACCESS_USERS, @@ -133,12 +134,16 @@ def is_access_management_activated(project_id: str) -> bool: return access_groups is not None and access_users is not None -def get_all_projects_by_user(organization_id) -> List[Project]: +def get_all_projects_by_user(organization_id: str) -> List[Project]: projects = project.get_all_by_user_organization_id(organization_id) project_dicts = sql_alchemy_to_dict( projects, column_whitelist=ALL_PROJECTS_WHITELIST ) + all_integration_project_ids = { + i.project_id for i in integration_db_co.get_all_in_org(organization_id) + } + for p in project_dicts: user_id = p["created_by"] names, mail = kratos.resolve_user_name_and_email_by_id(user_id) @@ -155,10 +160,7 @@ def get_all_projects_by_user(organization_id) -> List[Project]: else: p["num_data_scale_uploaded"] = record.get_count_scale_uploaded(p["id"]) - p["is_integration_project"] = any( - p["name"].startswith(f"[{integration_type.value}]") - for integration_type in enums.CognitionIntegrationType - ) + p["is_integration_project"] = p["id"] in all_integration_project_ids del p["created_by"] diff --git a/submodules/model b/submodules/model index 61ee231c..caf2c60b 160000 --- a/submodules/model +++ b/submodules/model @@ -1 +1 @@ -Subproject commit 61ee231c61dc3509b79b6fc738f5871e77911ecb +Subproject commit caf2c60bdc7d5a0ca3b91f3e148088fcc8be5f8c From 9e2974888e5e64d9a67fd048ada33345f3e0c67c Mon Sep 17 00:00:00 2001 From: JWittmeyer Date: Wed, 9 Jul 2025 11:21:09 +0200 Subject: [PATCH 10/11] PR comments --- controller/misc/manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controller/misc/manager.py b/controller/misc/manager.py index 29be80bd..946eb1bb 100644 --- a/controller/misc/manager.py +++ b/controller/misc/manager.py @@ -93,7 +93,7 @@ def finalize_customer_buttons( for e in buttons: e[key_name] = name_lookup[str(e[key])] e[key_name] = ( - (e[key_name]["first"] or "" + " " + e[key_name]["last"] or "") + (e[key_name].get("first", "") + " " + e[key_name].get("last", "")) if e[key_name] else "Unknown" ) From b604f1ae13e97f8d4e30c0975d7de354b9e97825 Mon Sep 17 00:00:00 2001 From: JWittmeyer Date: Wed, 9 Jul 2025 13:33:40 +0200 Subject: [PATCH 11/11] Submodule update --- submodules/model | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submodules/model b/submodules/model index caf2c60b..c5663e9c 160000 --- a/submodules/model +++ b/submodules/model @@ -1 +1 @@ -Subproject commit caf2c60bdc7d5a0ca3b91f3e148088fcc8be5f8c +Subproject commit c5663e9c855c30b431d106811a8330301c84eb24