diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/cf8f743fd0b7_add_indexes.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/cf8f743fd0b7_add_indexes.py new file mode 100644 index 000000000000..401630618622 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/cf8f743fd0b7_add_indexes.py @@ -0,0 +1,90 @@ +"""add indexes + +Revision ID: cf8f743fd0b7 +Revises: 48604dfdc5f4 +Create Date: 2025-04-04 09:46:38.853675+00:00 + +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "cf8f743fd0b7" +down_revision = "48604dfdc5f4" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_index( + "idx_project_to_groups_gid", "project_to_groups", ["gid"], unique=False + ) + op.create_index( + "idx_projects_last_change_date_desc", + "projects", + ["last_change_date"], + unique=False, + postgresql_using="btree", + postgresql_ops={"last_change_date": "DESC"}, + ) + op.create_index( + "ix_projects_partial_type", + "projects", + ["type"], + unique=False, + postgresql_where=sa.text("type = 'TEMPLATE'"), + ) + op.create_index( + "idx_project_to_folders_project_uuid", + "projects_to_folders", + ["project_uuid"], + unique=False, + ) + op.create_index( + "idx_project_to_folders_user_id", + "projects_to_folders", + ["user_id"], + unique=False, + ) + op.create_index( + "idx_projects_to_products_product_name", + "projects_to_products", + ["product_name"], + unique=False, + ) + op.create_index( + "idx_workspaces_access_rights_gid", + "workspaces_access_rights", + ["gid"], + unique=False, + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index( + "idx_workspaces_access_rights_gid", table_name="workspaces_access_rights" + ) + op.drop_index( + "idx_projects_to_products_product_name", table_name="projects_to_products" + ) + op.drop_index("idx_project_to_folders_user_id", table_name="projects_to_folders") + op.drop_index( + "idx_project_to_folders_project_uuid", table_name="projects_to_folders" + ) + op.drop_index( + "ix_projects_partial_type", + table_name="projects", + postgresql_where=sa.text("type = 'TEMPLATE'"), + ) + op.drop_index( + "idx_projects_last_change_date_desc", + table_name="projects", + postgresql_using="btree", + postgresql_ops={"last_change_date": "DESC"}, + ) + op.drop_index("idx_project_to_groups_gid", table_name="project_to_groups") + # ### end Alembic commands ### diff --git a/packages/postgres-database/src/simcore_postgres_database/models/project_to_groups.py b/packages/postgres-database/src/simcore_postgres_database/models/project_to_groups.py index 162a51d4d24d..4ae75fa40360 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/project_to_groups.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/project_to_groups.py @@ -60,4 +60,5 @@ column_created_datetime(timezone=True), column_modified_datetime(timezone=True), sa.UniqueConstraint("project_uuid", "gid"), + sa.Index("idx_project_to_groups_gid", "gid"), ) diff --git a/packages/postgres-database/src/simcore_postgres_database/models/projects.py b/packages/postgres-database/src/simcore_postgres_database/models/projects.py index abf853735713..9f903116b3f6 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/projects.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/projects.py @@ -1,6 +1,5 @@ -""" Projects table +"""Projects table""" -""" import enum import sqlalchemy as sa @@ -171,6 +170,20 @@ class ProjectType(enum.Enum): server_default=sa.text("'{}'::jsonb"), doc="DEPRECATED: Read/write/delete access rights of each group (gid) on this project", ), + ### INDEXES ---------------------------- + sa.Index( + "idx_projects_last_change_date_desc", + "last_change_date", + postgresql_using="btree", + postgresql_ops={"last_change_date": "DESC"}, + ), +) + +# We define the partial index +sa.Index( + "ix_projects_partial_type", + projects.c.type, + postgresql_where=(projects.c.type == ProjectType.TEMPLATE), ) diff --git a/packages/postgres-database/src/simcore_postgres_database/models/projects_to_folders.py b/packages/postgres-database/src/simcore_postgres_database/models/projects_to_folders.py index f86725a81194..34c242219367 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/projects_to_folders.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/projects_to_folders.py @@ -42,4 +42,6 @@ column_created_datetime(timezone=True), column_modified_datetime(timezone=True), sa.UniqueConstraint("project_uuid", "folder_id", "user_id"), + sa.Index("idx_project_to_folders_project_uuid", "project_uuid"), + sa.Index("idx_project_to_folders_user_id", "user_id"), ) diff --git a/packages/postgres-database/src/simcore_postgres_database/models/projects_to_products.py b/packages/postgres-database/src/simcore_postgres_database/models/projects_to_products.py index 47a430f0a000..0d8429ab3eea 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/projects_to_products.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/projects_to_products.py @@ -34,4 +34,5 @@ column_created_datetime(timezone=False), column_modified_datetime(timezone=False), sa.UniqueConstraint("project_uuid", "product_name"), + sa.Index("idx_projects_to_products_product_name", "product_name"), ) diff --git a/packages/postgres-database/src/simcore_postgres_database/models/workspaces_access_rights.py b/packages/postgres-database/src/simcore_postgres_database/models/workspaces_access_rights.py index 2a247fb477f8..6bc88d07338d 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/workspaces_access_rights.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/workspaces_access_rights.py @@ -57,4 +57,5 @@ column_created_datetime(timezone=True), column_modified_datetime(timezone=True), sa.UniqueConstraint("workspace_id", "gid"), + sa.Index("idx_workspaces_access_rights_gid", "gid"), ) diff --git a/services/storage/src/simcore_service_storage/modules/db/access_layer.py b/services/storage/src/simcore_service_storage/modules/db/access_layer.py index 0f70f0e959af..9e38e9cbf5ef 100644 --- a/services/storage/src/simcore_service_storage/modules/db/access_layer.py +++ b/services/storage/src/simcore_service_storage/modules/db/access_layer.py @@ -50,7 +50,6 @@ ) from simcore_postgres_database.storage_models import file_meta_data, user_to_groups from simcore_postgres_database.utils_repos import pass_or_acquire_connection -from simcore_postgres_database.utils_sql import assemble_array_groups from sqlalchemy.ext.asyncio import AsyncConnection from ...exceptions.errors import InvalidFileIdentifierError @@ -89,112 +88,104 @@ def _aggregate_access_rights( return AccessRights.none() -access_rights_subquery = ( - sa.select( - project_to_groups.c.project_uuid, - sa.func.jsonb_object_agg( - project_to_groups.c.gid, - sa.func.jsonb_build_object( - "read", - project_to_groups.c.read, - "write", - project_to_groups.c.write, - "delete", - project_to_groups.c.delete, - ), +def my_private_workspace_access_rights_subquery(user_group_ids: list[GroupID]): + return ( + sa.select( + project_to_groups.c.project_uuid, + sa.func.jsonb_object_agg( + project_to_groups.c.gid, + sa.func.jsonb_build_object( + "read", + project_to_groups.c.read, + "write", + project_to_groups.c.write, + "delete", + project_to_groups.c.delete, + ), + ).label("access_rights"), + ) + .where( + (project_to_groups.c.read) # Filters out entries where "read" is False + & ( + project_to_groups.c.gid.in_(user_group_ids) + ) # Filters gid to be in user_groups ) - .filter(project_to_groups.c.read) # Filters out entries where "read" is False - .label("access_rights"), - ).group_by(project_to_groups.c.project_uuid) -).subquery("access_rights_subquery") - - -workspace_access_rights_subquery = ( - sa.select( - workspaces_access_rights.c.workspace_id, - sa.func.jsonb_object_agg( - workspaces_access_rights.c.gid, - sa.func.jsonb_build_object( - "read", - workspaces_access_rights.c.read, - "write", - workspaces_access_rights.c.write, - "delete", - workspaces_access_rights.c.delete, - ), + .group_by(project_to_groups.c.project_uuid) + ).subquery("my_access_rights_subquery") + + +def my_shared_workspace_access_rights_subquery(user_group_ids: list[GroupID]): + return ( + sa.select( + workspaces_access_rights.c.workspace_id, + sa.func.jsonb_object_agg( + workspaces_access_rights.c.gid, + sa.func.jsonb_build_object( + "read", + workspaces_access_rights.c.read, + "write", + workspaces_access_rights.c.write, + "delete", + workspaces_access_rights.c.delete, + ), + ).label("access_rights"), + ) + .where( + ( + workspaces_access_rights.c.read + ) # Filters out entries where "read" is False + & ( + workspaces_access_rights.c.gid.in_(user_group_ids) + ) # Filters gid to be in user_groups ) - .filter(workspaces_access_rights.c.read) - .label("access_rights"), - ).group_by(workspaces_access_rights.c.workspace_id) -).subquery("workspace_access_rights_subquery") + .group_by(workspaces_access_rights.c.workspace_id) + ).subquery("my_workspace_access_rights_subquery") -async def _list_projects_access_rights( +async def _list_user_projects_access_rights_with_read_access( connection: AsyncConnection, user_id: UserID -) -> dict[ProjectID, AccessRights]: +) -> list[ProjectID]: """ Returns access-rights of user (user_id) over all OWNED or SHARED projects """ user_group_ids: list[GroupID] = await _get_user_groups_ids(connection, user_id) + _my_access_rights_subquery = my_private_workspace_access_rights_subquery( + user_group_ids + ) private_workspace_query = ( sa.select( projects.c.uuid, - access_rights_subquery.c.access_rights, - ) - .select_from(projects.join(access_rights_subquery, isouter=True)) - .where( - ( - (projects.c.prj_owner == user_id) - | sa.text( - f"jsonb_exists_any(access_rights_subquery.access_rights, {assemble_array_groups(user_group_ids)})" - ) - ) - & (projects.c.workspace_id.is_(None)) ) + .select_from(projects.join(_my_access_rights_subquery)) + .where(projects.c.workspace_id.is_(None)) + ) + + _my_workspace_access_rights_subquery = my_shared_workspace_access_rights_subquery( + user_group_ids ) shared_workspace_query = ( - sa.select( - projects.c.uuid, - workspace_access_rights_subquery.c.access_rights, - ) + sa.select(projects.c.uuid) .select_from( projects.join( - workspace_access_rights_subquery, + _my_workspace_access_rights_subquery, projects.c.workspace_id - == workspace_access_rights_subquery.c.workspace_id, + == _my_workspace_access_rights_subquery.c.workspace_id, ) ) - .where( - ( - sa.text( - f"jsonb_exists_any(workspace_access_rights_subquery.access_rights, {assemble_array_groups(user_group_ids)})" - ) - ) - & (projects.c.workspace_id.is_not(None)) - ) + .where(projects.c.workspace_id.is_not(None)) ) combined_query = sa.union_all(private_workspace_query, shared_workspace_query) - projects_access_rights = {} + projects_access_rights = [] async for row in await connection.stream(combined_query): - assert isinstance(row.access_rights, dict) # nosec assert isinstance(row.uuid, str) # nosec - if row.access_rights: - # NOTE: access_rights should be direclty filtered from result in stm instead calling again user_group_ids - projects_access_rights[ProjectID(row.uuid)] = _aggregate_access_rights( - row.access_rights, user_group_ids - ) - - else: - # backwards compatibility - # - no access_rights defined BUT project is owned - projects_access_rights[ProjectID(row.uuid)] = AccessRights.all() + projects_access_rights.append(ProjectID(row.uuid)) return projects_access_rights @@ -213,44 +204,40 @@ async def get_project_access_rights( async with pass_or_acquire_connection(self.db_engine, connection) as conn: user_group_ids = await _get_user_groups_ids(conn, user_id) + _my_access_rights_subquery = my_private_workspace_access_rights_subquery( + user_group_ids + ) private_workspace_query = ( sa.select( projects.c.prj_owner, - access_rights_subquery.c.access_rights, + _my_access_rights_subquery.c.access_rights, ) - .select_from(projects.join(access_rights_subquery, isouter=True)) + .select_from(projects.join(_my_access_rights_subquery)) .where( (projects.c.uuid == f"{project_id}") - & ( - (projects.c.prj_owner == user_id) - | sa.text( - f"jsonb_exists_any(access_rights_subquery.access_rights, {assemble_array_groups(user_group_ids)})" - ) - ) & (projects.c.workspace_id.is_(None)) ) ) + _my_workspace_access_rights_subquery = ( + my_shared_workspace_access_rights_subquery(user_group_ids) + ) + shared_workspace_query = ( sa.select( projects.c.prj_owner, - workspace_access_rights_subquery.c.access_rights, + _my_workspace_access_rights_subquery.c.access_rights, ) .select_from( projects.join( - workspace_access_rights_subquery, + _my_workspace_access_rights_subquery, projects.c.workspace_id - == workspace_access_rights_subquery.c.workspace_id, + == _my_workspace_access_rights_subquery.c.workspace_id, ) ) .where( (projects.c.uuid == f"{project_id}") - & ( - sa.text( - f"jsonb_exists_any(workspace_access_rights_subquery.access_rights, {assemble_array_groups(user_group_ids)})" - ) - ) & (projects.c.workspace_id.is_not(None)) ) ) @@ -358,5 +345,6 @@ async def get_readable_project_ids( ) -> list[ProjectID]: """Returns a list of projects where user has granted read-access""" async with pass_or_acquire_connection(self.db_engine, connection) as conn: - projects_access_rights = await _list_projects_access_rights(conn, user_id) - return [pid for pid, access in projects_access_rights.items() if access.read] + return await _list_user_projects_access_rights_with_read_access( + conn, user_id + ) diff --git a/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py b/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py index f20360d1cc02..0caa06d993bd 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py +++ b/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py @@ -188,14 +188,11 @@ async def list_projects_full_search(request: web.Request): if not query_params.filters: query_params.filters = ProjectFilters() - tag_ids_list = query_params.tag_ids_list() - projects, total_number_of_projects = await _crud_api_read.list_projects_full_depth( request.app, user_id=req_ctx.user_id, product_name=req_ctx.product_name, trashed=query_params.filters.trashed, - tag_ids_list=tag_ids_list, search_by_multi_columns=query_params.text, search_by_project_name=query_params.filters.search_by_project_name, offset=query_params.offset, diff --git a/services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py b/services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py index 45c0837ec03c..571c778057fd 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py +++ b/services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py @@ -175,7 +175,6 @@ async def list_projects_full_depth( product_name: str, # attrs filter trashed: bool | None, - tag_ids_list: list[int], # pagination offset: NonNegativeInt, limit: int, @@ -199,7 +198,6 @@ async def list_projects_full_depth( folder_query=FolderQuery(folder_scope=FolderScope.ALL), filter_trashed=trashed, filter_by_services=user_available_services, - filter_tag_ids_list=tag_ids_list, filter_by_project_type=ProjectType.STANDARD, search_by_multi_columns=search_by_multi_columns, search_by_project_name=search_by_project_name, diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py index e322c947a1af..3106b7a42c94 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py @@ -17,6 +17,7 @@ from aiopg.sa.result import ResultProxy, RowProxy from models_library.basic_types import IDStr from models_library.folders import FolderQuery, FolderScope +from models_library.groups import GroupID from models_library.products import ProductName from models_library.projects import ProjectID, ProjectIDStr from models_library.projects_comments import CommentID, ProjectsCommentsDB @@ -79,7 +80,6 @@ ANY_USER_ID_SENTINEL, BaseProjectDB, ProjectAccessRights, - assemble_array_groups, convert_to_db_names, convert_to_schema_names, create_project_access_rights, @@ -354,9 +354,8 @@ def _create_private_workspace_query( product_name: ProductName, user_id: UserID, workspace_query: WorkspaceQuery, - project_tags_subquery: sql.Subquery, is_search_by_multi_columns: bool, - user_groups: list[RowProxy], + user_groups: list[GroupID], ) -> sql.Select | None: private_workspace_query = None if workspace_query.workspace_scope is not WorkspaceScope.SHARED: @@ -378,13 +377,41 @@ def _create_private_workspace_query( "delete", project_to_groups.c.delete, ), - ) - .filter( - project_to_groups.c.read # Filters out entries where "read" is False - ) - .label("access_rights"), - ).group_by(project_to_groups.c.project_uuid) - ).subquery("access_rights_subquery") + ).label("access_rights"), + ) + .where( + project_to_groups.c.project_uuid == projects.c.uuid + ) # Correlate with main query + .where(project_to_groups.c.read) + .group_by(project_to_groups.c.project_uuid) + .lateral() # Critical for per-row execution + ) + + my_access_rights_subquery = ( + sa.select( + project_to_groups.c.project_uuid, + sa.func.jsonb_object_agg( + project_to_groups.c.gid, + sa.func.jsonb_build_object( + "read", + project_to_groups.c.read, + "write", + project_to_groups.c.write, + "delete", + project_to_groups.c.delete, + ), + ).label("access_rights"), + ) + .where( + ( + project_to_groups.c.read + ) # Filters out entries where "read" is False + & ( + project_to_groups.c.gid.in_(user_groups) + ) # Filters gid to be in user_groups + ) + .group_by(project_to_groups.c.project_uuid) + ).subquery("my_access_rights_subquery") private_workspace_query = ( sa.select( @@ -393,13 +420,9 @@ def _create_private_workspace_query( access_rights_subquery.c.access_rights, projects_to_products.c.product_name, projects_to_folders.c.folder_id, - sa.func.coalesce( - project_tags_subquery.c.tags, - sa.cast(sa.text("'{}'"), sa.ARRAY(sa.Integer)), - ).label("tags"), ) .select_from( - projects.join(access_rights_subquery, isouter=True) + projects.join(my_access_rights_subquery) .join(projects_to_products) .join( projects_to_folders, @@ -409,21 +432,19 @@ def _create_private_workspace_query( ), isouter=True, ) - .join(project_tags_subquery, isouter=True) + .join( + access_rights_subquery, + access_rights_subquery.c.project_uuid == projects.c.uuid, + ) ) .where( - ( - (projects.c.prj_owner == user_id) - | sa.text( - f"jsonb_exists_any(access_rights_subquery.access_rights, {assemble_array_groups(user_groups)})" - ) - ) - & (projects.c.workspace_id.is_(None)) # <-- Private workspace + (projects.c.workspace_id.is_(None)) # <-- Private workspace & (projects_to_products.c.product_name == product_name) ) ) + assert ( # nosec - access_rights_subquery.description == "access_rights_subquery" + my_access_rights_subquery.description == "my_access_rights_subquery" ) if is_search_by_multi_columns: @@ -438,9 +459,8 @@ def _create_shared_workspace_query( *, product_name: ProductName, workspace_query: WorkspaceQuery, - project_tags_subquery: sql.Subquery, - user_groups: list[RowProxy], is_search_by_multi_columns: bool, + user_groups: list[GroupID], ) -> sql.Select | None: if workspace_query.workspace_scope is not WorkspaceScope.PRIVATE: @@ -450,6 +470,31 @@ def _create_shared_workspace_query( ) workspace_access_rights_subquery = ( + ( + sa.select( + workspaces_access_rights.c.workspace_id, + sa.func.jsonb_object_agg( + workspaces_access_rights.c.gid, + sa.func.jsonb_build_object( + "read", + workspaces_access_rights.c.read, + "write", + workspaces_access_rights.c.write, + "delete", + workspaces_access_rights.c.delete, + ), + ).label("access_rights"), + ) + .where( + workspaces_access_rights.c.read, + ) + .group_by(workspaces_access_rights.c.workspace_id) + ) + .subquery("workspace_access_rights_subquery") + .lateral() + ) + + my_workspace_access_rights_subquery = ( sa.select( workspaces_access_rights.c.workspace_id, sa.func.jsonb_object_agg( @@ -462,11 +507,14 @@ def _create_shared_workspace_query( "delete", workspaces_access_rights.c.delete, ), - ) - .filter(workspaces_access_rights.c.read) - .label("access_rights"), - ).group_by(workspaces_access_rights.c.workspace_id) - ).subquery("workspace_access_rights_subquery") + ).label("access_rights"), + ) + .where( + workspaces_access_rights.c.read, + workspaces_access_rights.c.gid.in_(user_groups), + ) + .group_by(workspaces_access_rights.c.workspace_id) + ).subquery("my_workspace_access_rights_subquery") shared_workspace_query = ( sa.select( @@ -475,16 +523,12 @@ def _create_shared_workspace_query( workspace_access_rights_subquery.c.access_rights, projects_to_products.c.product_name, projects_to_folders.c.folder_id, - sa.func.coalesce( - project_tags_subquery.c.tags, - sa.cast(sa.text("'{}'"), sa.ARRAY(sa.Integer)), - ).label("tags"), ) .select_from( projects.join( - workspace_access_rights_subquery, + my_workspace_access_rights_subquery, projects.c.workspace_id - == workspace_access_rights_subquery.c.workspace_id, + == my_workspace_access_rights_subquery.c.workspace_id, ) .join(projects_to_products) .join( @@ -495,20 +539,17 @@ def _create_shared_workspace_query( ), isouter=True, ) - .join(project_tags_subquery, isouter=True) - ) - .where( - ( - sa.text( - f"jsonb_exists_any(workspace_access_rights_subquery.access_rights, {assemble_array_groups(user_groups)})" - ) + .join( + workspace_access_rights_subquery, + projects.c.workspace_id + == workspace_access_rights_subquery.c.workspace_id, ) - & (projects_to_products.c.product_name == product_name) ) + .where(projects_to_products.c.product_name == product_name) ) assert ( # nosec - workspace_access_rights_subquery.description - == "workspace_access_rights_subquery" + my_workspace_access_rights_subquery.description + == "my_workspace_access_rights_subquery" ) if workspace_query.workspace_scope == WorkspaceScope.ALL: @@ -541,9 +582,7 @@ def _create_attributes_filters( filter_trashed: bool | None, search_by_multi_columns: str | None, search_by_project_name: str | None, - filter_tag_ids_list: list[int] | None, folder_query: FolderQuery, - project_tags_subquery: sql.Subquery, ) -> list[ColumnElement]: attributes_filters: list[ColumnElement] = [] @@ -581,14 +620,6 @@ def _create_attributes_filters( projects.c.name.like(f"%{search_by_project_name}%") ) - if filter_tag_ids_list: - attributes_filters.append( - sa.func.coalesce( - project_tags_subquery.c.tags, - sa.cast(sa.text("'{}'"), sa.ARRAY(sa.Integer)), - ).op("@>")(filter_tag_ids_list) - ) - if folder_query.folder_scope is not FolderScope.ALL: if folder_query.folder_scope == FolderScope.SPECIFIC: attributes_filters.append( @@ -614,7 +645,6 @@ async def list_projects_dicts( # pylint: disable=too-many-arguments,too-many-st filter_published: bool | None = None, filter_hidden: bool | None = False, filter_trashed: bool | None = False, - filter_tag_ids_list: list[int] | None = None, # search search_by_multi_columns: str | None = None, search_by_project_name: str | None = None, @@ -624,20 +654,11 @@ async def list_projects_dicts( # pylint: disable=too-many-arguments,too-many-st # order order_by: OrderBy = DEFAULT_ORDER_BY, ) -> tuple[list[ProjectDict], list[ProjectType], int]: - - if filter_tag_ids_list is None: - filter_tag_ids_list = [] - async with self.engine.acquire() as conn: - user_groups: list[RowProxy] = await self._list_user_groups(conn, user_id) - project_tags_subquery = ( - sa.select( - projects_tags.c.project_id, - sa.func.array_agg(projects_tags.c.tag_id).label("tags"), - ) - .where(projects_tags.c.project_id.is_not(None)) - .group_by(projects_tags.c.project_id) - ).subquery("project_tags_subquery") + user_groups_proxy: list[RowProxy] = await self._list_user_groups( + conn, user_id + ) + user_groups: list[GroupID] = [group.gid for group in user_groups_proxy] ### # Private workspace query @@ -647,7 +668,6 @@ async def list_projects_dicts( # pylint: disable=too-many-arguments,too-many-st product_name=product_name, user_id=user_id, workspace_query=workspace_query, - project_tags_subquery=project_tags_subquery, is_search_by_multi_columns=search_by_multi_columns is not None, user_groups=user_groups, ) @@ -658,9 +678,8 @@ async def list_projects_dicts( # pylint: disable=too-many-arguments,too-many-st shared_workspace_query = self._create_shared_workspace_query( product_name=product_name, workspace_query=workspace_query, - project_tags_subquery=project_tags_subquery, - user_groups=user_groups, is_search_by_multi_columns=search_by_multi_columns is not None, + user_groups=user_groups, ) ### @@ -674,9 +693,7 @@ async def list_projects_dicts( # pylint: disable=too-many-arguments,too-many-st filter_trashed=filter_trashed, search_by_multi_columns=search_by_multi_columns, search_by_project_name=search_by_project_name, - filter_tag_ids_list=filter_tag_ids_list, folder_query=folder_query, - project_tags_subquery=project_tags_subquery, ) ### @@ -709,11 +726,13 @@ async def list_projects_dicts( # pylint: disable=too-many-arguments,too-many-st if order_by.direction == OrderDirection.ASC: combined_query = combined_query.order_by( - sa.asc(getattr(projects.c, order_by.field)) + sa.asc(getattr(projects.c, order_by.field)), + projects.c.id, ) else: combined_query = combined_query.order_by( - sa.desc(getattr(projects.c, order_by.field)) + sa.desc(getattr(projects.c, order_by.field)), + projects.c.id, ) prjs, prj_types = await self._execute_without_permission_check( diff --git a/services/web/server/src/simcore_service_webserver/projects/_trash_service.py b/services/web/server/src/simcore_service_webserver/projects/_trash_service.py index a70a52937bb1..ca48c60d878a 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_trash_service.py +++ b/services/web/server/src/simcore_service_webserver/projects/_trash_service.py @@ -169,7 +169,6 @@ async def list_explicitly_trashed_projects( user_id=user_id, product_name=product_name, trashed=True, - tag_ids_list=[], offset=page_params.offset, limit=page_params.limit, order_by=OrderBy(field=IDStr("trashed"), direction=OrderDirection.ASC), diff --git a/services/web/server/tests/unit/isolated/test_projects__db_utils.py b/services/web/server/tests/unit/isolated/test_projects__db_utils.py index 06631e73a4ba..7f42f14f23b4 100644 --- a/services/web/server/tests/unit/isolated/test_projects__db_utils.py +++ b/services/web/server/tests/unit/isolated/test_projects__db_utils.py @@ -18,7 +18,6 @@ from models_library.utils.fastapi_encoders import jsonable_encoder from simcore_service_webserver.projects._projects_repository_legacy import ( ProjectAccessRights, - assemble_array_groups, convert_to_db_names, convert_to_schema_names, create_project_access_rights, @@ -28,6 +27,7 @@ from simcore_service_webserver.projects._projects_repository_legacy_utils import ( DB_EXCLUSIVE_COLUMNS, SCHEMA_NON_NULL_KEYS, + assemble_array_groups, ) from simcore_service_webserver.projects.exceptions import ( NodeNotFoundError, diff --git a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__list_projects_full_search.py b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__list_projects_full_search.py index c71cdf4fb406..f3846d1650dd 100644 --- a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__list_projects_full_search.py +++ b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__list_projects_full_search.py @@ -202,31 +202,34 @@ async def test__list_projects_full_search_with_query_parameters( assert len(data) == 1 assert data[0]["uuid"] == project["uuid"] - # Full search with tag_ids - base_url = client.app.router["list_projects_full_search"].url_for() - url = base_url.with_query({"text": "Orion", "tag_ids": "1,2"}) - resp = await client.get(f"{url}") - data, _ = await assert_status(resp, status.HTTP_200_OK) - assert len(data) == 0 - - # Create tag - url = client.app.router["create_tag"].url_for() - resp = await client.post( - f"{url}", json={"name": "tag1", "description": "description1", "color": "#f00"} - ) - added_tag, _ = await assert_status(resp, status.HTTP_201_CREATED) - - # Add tag to study - url = client.app.router["add_project_tag"].url_for( - project_uuid=project["uuid"], tag_id=str(added_tag.get("id")) - ) - resp = await client.post(f"{url}") - data, _ = await assert_status(resp, status.HTTP_200_OK) - - # Full search with tag_ids - base_url = client.app.router["list_projects_full_search"].url_for() - url = base_url.with_query({"text": "Orion", "tag_ids": f"{added_tag['id']}"}) - resp = await client.get(f"{url}") - data, _ = await assert_status(resp, status.HTTP_200_OK) - assert len(data) == 1 - assert data[0]["uuid"] == project["uuid"] + # NOTE: MD: To improve the listing project performance https://github.com/ITISFoundation/osparc-simcore/pull/7475 + # we are not using the tag_ids in the full search (https://github.com/ITISFoundation/osparc-simcore/issues/7478) + + # # Full search with tag_ids + # base_url = client.app.router["list_projects_full_search"].url_for() + # url = base_url.with_query({"text": "Orion", "tag_ids": "1,2"}) + # resp = await client.get(f"{url}") + # data, _ = await assert_status(resp, status.HTTP_200_OK) + # assert len(data) == 0 + + # # Create tag + # url = client.app.router["create_tag"].url_for() + # resp = await client.post( + # f"{url}", json={"name": "tag1", "description": "description1", "color": "#f00"} + # ) + # added_tag, _ = await assert_status(resp, status.HTTP_201_CREATED) + + # # Add tag to study + # url = client.app.router["add_project_tag"].url_for( + # project_uuid=project["uuid"], tag_id=str(added_tag.get("id")) + # ) + # resp = await client.post(f"{url}") + # data, _ = await assert_status(resp, status.HTTP_200_OK) + + # # Full search with tag_ids + # base_url = client.app.router["list_projects_full_search"].url_for() + # url = base_url.with_query({"text": "Orion", "tag_ids": f"{added_tag['id']}"}) + # resp = await client.get(f"{url}") + # data, _ = await assert_status(resp, status.HTTP_200_OK) + # assert len(data) == 1 + # assert data[0]["uuid"] == project["uuid"]