♻️🚨 Improve DB query listing_projects performance (🗃️)

packages/postgres-database/src/simcore_postgres_database/migration/versions/cf8f743fd0b7_add_indexes.py

@@ -0,0 +1,90 @@
+"""add indexes
+
+Revision ID: cf8f743fd0b7
+Revises: 48604dfdc5f4
+Create Date: 2025-04-04 09:46:38.853675+00:00
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "cf8f743fd0b7"
+down_revision = "48604dfdc5f4"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_index(
+        "idx_project_to_groups_gid", "project_to_groups", ["gid"], unique=False
+    )
+    op.create_index(
+        "idx_projects_last_change_date_desc",
+        "projects",
+        ["last_change_date"],
+        unique=False,
+        postgresql_using="btree",
+        postgresql_ops={"last_change_date": "DESC"},
+    )
+    op.create_index(
+        "ix_projects_partial_type",
+        "projects",
+        ["type"],
+        unique=False,
+        postgresql_where=sa.text("type = 'TEMPLATE'"),
+    )
+    op.create_index(
+        "idx_project_to_folders_project_uuid",
+        "projects_to_folders",
+        ["project_uuid"],
+        unique=False,
+    )
+    op.create_index(
+        "idx_project_to_folders_user_id",
+        "projects_to_folders",
+        ["user_id"],
+        unique=False,
+    )
+    op.create_index(
+        "idx_projects_to_products_product_name",
+        "projects_to_products",
+        ["product_name"],
+        unique=False,
+    )
+    op.create_index(
+        "idx_workspaces_access_rights_gid",
+        "workspaces_access_rights",
+        ["gid"],
+        unique=False,
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(
+        "idx_workspaces_access_rights_gid", table_name="workspaces_access_rights"
+    )
+    op.drop_index(
+        "idx_projects_to_products_product_name", table_name="projects_to_products"
+    )
+    op.drop_index("idx_project_to_folders_user_id", table_name="projects_to_folders")
+    op.drop_index(
+        "idx_project_to_folders_project_uuid", table_name="projects_to_folders"
+    )
+    op.drop_index(
+        "ix_projects_partial_type",
+        table_name="projects",
+        postgresql_where=sa.text("type = 'TEMPLATE'"),
+    )
+    op.drop_index(
+        "idx_projects_last_change_date_desc",
+        table_name="projects",
+        postgresql_using="btree",
+        postgresql_ops={"last_change_date": "DESC"},
+    )
+    op.drop_index("idx_project_to_groups_gid", table_name="project_to_groups")
+    # ### end Alembic commands ###

packages/postgres-database/src/simcore_postgres_database/models/project_to_groups.py

@@ -60,4 +60,5 @@
     column_created_datetime(timezone=True),
     column_modified_datetime(timezone=True),
     sa.UniqueConstraint("project_uuid", "gid"),
+    sa.Index("idx_project_to_groups_gid", "gid"),
 )

packages/postgres-database/src/simcore_postgres_database/models/projects.py

@@ -1,6 +1,5 @@
-""" Projects table
+"""Projects table"""
 
-"""
 import enum
 
 import sqlalchemy as sa
@@ -171,6 +170,20 @@ class ProjectType(enum.Enum):
         server_default=sa.text("'{}'::jsonb"),
         doc="DEPRECATED: Read/write/delete access rights of each group (gid) on this project",
     ),
+    ### INDEXES ----------------------------
+    sa.Index(
+        "idx_projects_last_change_date_desc",
+        "last_change_date",
+        postgresql_using="btree",
+        postgresql_ops={"last_change_date": "DESC"},
+    ),
+)
+
+# We define the partial index
+sa.Index(
+    "ix_projects_partial_type",
+    projects.c.type,
+    postgresql_where=(projects.c.type == ProjectType.TEMPLATE),
 )
 
 

packages/postgres-database/src/simcore_postgres_database/models/projects_to_folders.py

@@ -42,4 +42,6 @@
     column_created_datetime(timezone=True),
     column_modified_datetime(timezone=True),
     sa.UniqueConstraint("project_uuid", "folder_id", "user_id"),
+    sa.Index("idx_project_to_folders_project_uuid", "project_uuid"),
+    sa.Index("idx_project_to_folders_user_id", "user_id"),
 )

packages/postgres-database/src/simcore_postgres_database/models/projects_to_products.py

@@ -34,4 +34,5 @@
     column_created_datetime(timezone=False),
     column_modified_datetime(timezone=False),
     sa.UniqueConstraint("project_uuid", "product_name"),
+    sa.Index("idx_projects_to_products_product_name", "product_name"),
 )

packages/postgres-database/src/simcore_postgres_database/models/workspaces_access_rights.py

@@ -57,4 +57,5 @@
     column_created_datetime(timezone=True),
     column_modified_datetime(timezone=True),
     sa.UniqueConstraint("workspace_id", "gid"),
+    sa.Index("idx_workspaces_access_rights_gid", "gid"),
 )

services/storage/src/simcore_service_storage/modules/db/access_layer.py

@@ -50,7 +50,6 @@
 )
 from simcore_postgres_database.storage_models import file_meta_data, user_to_groups
 from simcore_postgres_database.utils_repos import pass_or_acquire_connection
-from simcore_postgres_database.utils_sql import assemble_array_groups
 from sqlalchemy.ext.asyncio import AsyncConnection
 
 from ...exceptions.errors import InvalidFileIdentifierError
@@ -89,112 +88,104 @@ def _aggregate_access_rights(
         return AccessRights.none()
 
 
-access_rights_subquery = (
-    sa.select(
-        project_to_groups.c.project_uuid,
-        sa.func.jsonb_object_agg(
-            project_to_groups.c.gid,
-            sa.func.jsonb_build_object(
-                "read",
-                project_to_groups.c.read,
-                "write",
-                project_to_groups.c.write,
-                "delete",
-                project_to_groups.c.delete,
-            ),
+def my_private_workspace_access_rights_subquery(user_group_ids: list[GroupID]):
+    return (
+        sa.select(
+            project_to_groups.c.project_uuid,
+            sa.func.jsonb_object_agg(
+                project_to_groups.c.gid,
+                sa.func.jsonb_build_object(
+                    "read",
+                    project_to_groups.c.read,
+                    "write",
+                    project_to_groups.c.write,
+                    "delete",
+                    project_to_groups.c.delete,
+                ),
+            ).label("access_rights"),
+        )
+        .where(
+            (project_to_groups.c.read)  # Filters out entries where "read" is False
+            & (
+                project_to_groups.c.gid.in_(user_group_ids)
+            )  # Filters gid to be in user_groups
         )
-        .filter(project_to_groups.c.read)  # Filters out entries where "read" is False
-        .label("access_rights"),
-    ).group_by(project_to_groups.c.project_uuid)
-).subquery("access_rights_subquery")
-
-
-workspace_access_rights_subquery = (
-    sa.select(
-        workspaces_access_rights.c.workspace_id,
-        sa.func.jsonb_object_agg(
-            workspaces_access_rights.c.gid,
-            sa.func.jsonb_build_object(
-                "read",
-                workspaces_access_rights.c.read,
-                "write",
-                workspaces_access_rights.c.write,
-                "delete",
-                workspaces_access_rights.c.delete,
-            ),
+        .group_by(project_to_groups.c.project_uuid)
+    ).subquery("my_access_rights_subquery")
+
+
+def my_shared_workspace_access_rights_subquery(user_group_ids: list[GroupID]):
+    return (
+        sa.select(
+            workspaces_access_rights.c.workspace_id,
+            sa.func.jsonb_object_agg(
+                workspaces_access_rights.c.gid,
+                sa.func.jsonb_build_object(
+                    "read",
+                    workspaces_access_rights.c.read,
+                    "write",
+                    workspaces_access_rights.c.write,
+                    "delete",
+                    workspaces_access_rights.c.delete,
+                ),
+            ).label("access_rights"),
+        )
+        .where(
+            (
+                workspaces_access_rights.c.read
+            )  # Filters out entries where "read" is False
+            & (
+                workspaces_access_rights.c.gid.in_(user_group_ids)
+            )  # Filters gid to be in user_groups
         )
-        .filter(workspaces_access_rights.c.read)
-        .label("access_rights"),
-    ).group_by(workspaces_access_rights.c.workspace_id)
-).subquery("workspace_access_rights_subquery")
+        .group_by(workspaces_access_rights.c.workspace_id)
+    ).subquery("my_workspace_access_rights_subquery")
 
 
-async def _list_projects_access_rights(
+async def _list_user_projects_access_rights_with_read_access(
     connection: AsyncConnection, user_id: UserID
-) -> dict[ProjectID, AccessRights]:
+) -> list[ProjectID]:
     """
     Returns access-rights of user (user_id) over all OWNED or SHARED projects
     """
 
     user_group_ids: list[GroupID] = await _get_user_groups_ids(connection, user_id)
+    _my_access_rights_subquery = my_private_workspace_access_rights_subquery(
+        user_group_ids
+    )
 
     private_workspace_query = (
         sa.select(
             projects.c.uuid,
-            access_rights_subquery.c.access_rights,
-        )
-        .select_from(projects.join(access_rights_subquery, isouter=True))
-        .where(
-            (
-                (projects.c.prj_owner == user_id)
-                | sa.text(
-                    f"jsonb_exists_any(access_rights_subquery.access_rights, {assemble_array_groups(user_group_ids)})"
-                )
-            )
-            & (projects.c.workspace_id.is_(None))
         )
+        .select_from(projects.join(_my_access_rights_subquery))
+        .where(projects.c.workspace_id.is_(None))
+    )
+
+    _my_workspace_access_rights_subquery = my_shared_workspace_access_rights_subquery(
+        user_group_ids
     )
 
     shared_workspace_query = (
-        sa.select(
-            projects.c.uuid,
-            workspace_access_rights_subquery.c.access_rights,
-        )
+        sa.select(projects.c.uuid)
         .select_from(
             projects.join(
-                workspace_access_rights_subquery,
+                _my_workspace_access_rights_subquery,
                 projects.c.workspace_id
-                == workspace_access_rights_subquery.c.workspace_id,
+                == _my_workspace_access_rights_subquery.c.workspace_id,
             )
         )
-        .where(
-            (
-                sa.text(
-                    f"jsonb_exists_any(workspace_access_rights_subquery.access_rights, {assemble_array_groups(user_group_ids)})"
-                )
-            )
-            & (projects.c.workspace_id.is_not(None))
-        )
+        .where(projects.c.workspace_id.is_not(None))
     )
 
     combined_query = sa.union_all(private_workspace_query, shared_workspace_query)
 
-    projects_access_rights = {}
+    projects_access_rights = []
 
     async for row in await connection.stream(combined_query):
-        assert isinstance(row.access_rights, dict)  # nosec
         assert isinstance(row.uuid, str)  # nosec
 
-        if row.access_rights:
-            # NOTE: access_rights should be direclty filtered from result in stm instead calling again user_group_ids
-            projects_access_rights[ProjectID(row.uuid)] = _aggregate_access_rights(
-                row.access_rights, user_group_ids
-            )
-
-        else:
-            # backwards compatibility
-            # - no access_rights defined BUT project is owned
-            projects_access_rights[ProjectID(row.uuid)] = AccessRights.all()
+        projects_access_rights.append(ProjectID(row.uuid))
 
     return projects_access_rights
 
@@ -213,44 +204,40 @@ async def get_project_access_rights(
 
         async with pass_or_acquire_connection(self.db_engine, connection) as conn:
             user_group_ids = await _get_user_groups_ids(conn, user_id)
+            _my_access_rights_subquery = my_private_workspace_access_rights_subquery(
+                user_group_ids
+            )
 
             private_workspace_query = (
                 sa.select(
                     projects.c.prj_owner,
-                    access_rights_subquery.c.access_rights,
+                    _my_access_rights_subquery.c.access_rights,
                 )
-                .select_from(projects.join(access_rights_subquery, isouter=True))
+                .select_from(projects.join(_my_access_rights_subquery))
                 .where(
                     (projects.c.uuid == f"{project_id}")
-                    & (
-                        (projects.c.prj_owner == user_id)
-                        | sa.text(
-                            f"jsonb_exists_any(access_rights_subquery.access_rights, {assemble_array_groups(user_group_ids)})"
-                        )
-                    )
                     & (projects.c.workspace_id.is_(None))
                 )
             )
 
+            _my_workspace_access_rights_subquery = (
+                my_shared_workspace_access_rights_subquery(user_group_ids)
+            )
+
             shared_workspace_query = (
                 sa.select(
                     projects.c.prj_owner,
-                    workspace_access_rights_subquery.c.access_rights,
+                    _my_workspace_access_rights_subquery.c.access_rights,
                 )
                 .select_from(
                     projects.join(
-                        workspace_access_rights_subquery,
+                        _my_workspace_access_rights_subquery,
                         projects.c.workspace_id
-                        == workspace_access_rights_subquery.c.workspace_id,
+                        == _my_workspace_access_rights_subquery.c.workspace_id,
                     )
                 )
                 .where(
                     (projects.c.uuid == f"{project_id}")
-                    & (
-                        sa.text(
-                            f"jsonb_exists_any(workspace_access_rights_subquery.access_rights, {assemble_array_groups(user_group_ids)})"
-                        )
-                    )
                     & (projects.c.workspace_id.is_not(None))
                 )
             )
@@ -358,5 +345,6 @@ async def get_readable_project_ids(
     ) -> list[ProjectID]:
         """Returns a list of projects where user has granted read-access"""
         async with pass_or_acquire_connection(self.db_engine, connection) as conn:
-            projects_access_rights = await _list_projects_access_rights(conn, user_id)
-        return [pid for pid, access in projects_access_rights.items() if access.read]
+            return await _list_user_projects_access_rights_with_read_access(
+                conn, user_id
+            )