Skip to content

♻️ speedup project listing 🚨 #7737

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 10 commits into from
Jun 2, 2025
Merged

♻️ speedup project listing 🚨 #7737

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
3a870a5
speedup project listing
matusdrobuliak66 May 22, 2025
f880b1f
Merge branch 'master' into improve-project-listing-db
matusdrobuliak66 May 29, 2025
dde02ee
Merge branch 'master' into improve-project-listing-db
matusdrobuliak66 May 30, 2025
097a18d
batch get project access rights sql query
matusdrobuliak66 Jun 1, 2025
0f9aa5c
batch get project access rights sql query
matusdrobuliak66 Jun 1, 2025
d9f465f
remove comments
matusdrobuliak66 Jun 1, 2025
dffe0c7
remove comments
matusdrobuliak66 Jun 1, 2025
1745552
final
matusdrobuliak66 Jun 1, 2025
e23333c
final improvement
matusdrobuliak66 Jun 1, 2025
9087517
Merge branch 'master' into improve-project-listing-db
matusdrobuliak66 Jun 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
109 changes: 106 additions & 3 deletions services/web/server/src/simcore_service_webserver/projects/_access_rights_repository.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,15 +1,27 @@
import sqlalchemy
import sqlalchemy as sa
from aiohttp import web
from aiopg.sa.engine import Engine
from models_library.projects import ProjectID
from models_library.groups import GroupID
from models_library.projects import ProjectID, ProjectIDStr
from models_library.users import UserID
from models_library.workspaces import WorkspaceID
from simcore_postgres_database.models.project_to_groups import project_to_groups
from simcore_postgres_database.models.projects import projects
from simcore_postgres_database.models.workspaces_access_rights import (
workspaces_access_rights,
)
from simcore_postgres_database.utils_repos import (
pass_or_acquire_connection,
)
from sqlalchemy.ext.asyncio import AsyncConnection

from ..db.plugin import get_asyncpg_engine
from .exceptions import ProjectNotFoundError


async def get_project_owner(engine: Engine, project_uuid: ProjectID) -> UserID:
async with engine.acquire() as connection:
stmt = sqlalchemy.select(projects.c.prj_owner).where(
stmt = sa.select(projects.c.prj_owner).where(
projects.c.uuid == f"{project_uuid}"
)

Expand All @@ -18,3 +30,94 @@ async def get_project_owner(engine: Engine, project_uuid: ProjectID) -> UserID:
raise ProjectNotFoundError(project_uuid=project_uuid)
assert isinstance(owner_id, int)
return owner_id


def _split_private_and_shared_projects(
projects_uuids_with_workspace_id: list[tuple[ProjectID, WorkspaceID | None]],
) -> tuple[list[ProjectID], dict[WorkspaceID, list[ProjectID]]]:
"""Splits project tuples into private project IDs and a mapping of workspace_id to project IDs."""
private_project_ids = []
workspace_to_project_ids: dict[WorkspaceID, list[ProjectID]] = {}
for pid, wid in projects_uuids_with_workspace_id:
if wid is None:
private_project_ids.append(pid)
else:
workspace_to_project_ids.setdefault(wid, []).append(pid)
return private_project_ids, workspace_to_project_ids


async def batch_get_project_access_rights(
app: web.Application,
connection: AsyncConnection | None = None,
*,
projects_uuids_with_workspace_id: list[tuple[ProjectID, WorkspaceID | None]],
) -> dict[ProjectIDStr, dict[GroupID, dict[str, bool]]]:
private_project_ids, workspace_to_project_ids = _split_private_and_shared_projects(
projects_uuids_with_workspace_id
)
shared_workspace_ids = set(workspace_to_project_ids.keys())
results = {}

async with pass_or_acquire_connection(get_asyncpg_engine(app), connection) as conn:
# Query private workspace projects
if private_project_ids:
private_query = (
sa.select(
project_to_groups.c.project_uuid,
sa.func.jsonb_object_agg(
project_to_groups.c.gid,
sa.func.jsonb_build_object(
"read",
project_to_groups.c.read,
"write",
project_to_groups.c.write,
"delete",
project_to_groups.c.delete,
),
).label("access_rights"),
)
.where(
project_to_groups.c.project_uuid.in_(
[f"{uuid}" for uuid in private_project_ids]
)
)
.group_by(project_to_groups.c.project_uuid)
)
private_result = await conn.stream(private_query)
async for row in private_result:
results[row.project_uuid] = row.access_rights

# Query shared workspace projects by workspace_id
if shared_workspace_ids:
shared_query = (
sa.select(
workspaces_access_rights.c.workspace_id,
sa.func.jsonb_object_agg(
workspaces_access_rights.c.gid,
sa.func.jsonb_build_object(
"read",
workspaces_access_rights.c.read,
"write",
workspaces_access_rights.c.write,
"delete",
workspaces_access_rights.c.delete,
),
).label("access_rights"),
)
.where(
workspaces_access_rights.c.workspace_id.in_(shared_workspace_ids)
)
.group_by(workspaces_access_rights.c.workspace_id)
)
shared_result = await conn.stream(shared_query)
workspace_access_rights_map = {}
async for row in shared_result:
workspace_access_rights_map[row.workspace_id] = row.access_rights
# Assign access rights to each project in the workspace
for wid, project_ids in workspace_to_project_ids.items():
access_rights = workspace_access_rights_map.get(wid)
if access_rights is not None:
for pid in project_ids:
results[f"{pid}"] = access_rights

return results
12 changes: 12 additions & 0 deletions services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
from ..folders import _folders_repository
from ..workspaces.api import check_user_workspace_access
from . import _projects_service
from ._access_rights_repository import batch_get_project_access_rights
from ._projects_repository import batch_get_trashed_by_primary_gid
from ._projects_repository_legacy import ProjectDBAPI
from .models import ProjectDict, ProjectTypeAPI
Expand Down Expand Up @@ -66,6 +67,14 @@ async def _aggregate_data_to_projects_from_other_sources(

_batch_update("trashed_by_primary_gid", trashed_by_primary_gid_values, db_projects)

# Add here get batch Project access rights
project_to_access_rights = await batch_get_project_access_rights(
app=app,
projects_uuids_with_workspace_id=[
(ProjectID(p["uuid"]), p["workspaceId"]) for p in db_projects
],
)

# udpating `project.state`
update_state_per_project = [
_projects_service.add_project_states_for_user(
Expand All @@ -81,6 +90,9 @@ async def _aggregate_data_to_projects_from_other_sources(
*update_state_per_project,
)

for project in updated_projects:
project["accessRights"] = project_to_access_rights[project["uuid"]]

return updated_projects


Expand Down
89 changes: 2 additions & 87 deletions services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -374,44 +374,8 @@ def _create_private_workspace_query(
WorkspaceScope.ALL,
)

access_rights_subquery = (
sa.select(
project_to_groups.c.project_uuid,
sa.func.jsonb_object_agg(
project_to_groups.c.gid,
sa.func.jsonb_build_object(
"read",
project_to_groups.c.read,
"write",
project_to_groups.c.write,
"delete",
project_to_groups.c.delete,
),
).label("access_rights"),
)
.where(
project_to_groups.c.project_uuid == projects.c.uuid
) # Correlate with main query
.where(project_to_groups.c.read)
.group_by(project_to_groups.c.project_uuid)
.lateral() # Critical for per-row execution
)

my_access_rights_subquery = (
sa.select(
project_to_groups.c.project_uuid,
sa.func.jsonb_object_agg(
project_to_groups.c.gid,
sa.func.jsonb_build_object(
"read",
project_to_groups.c.read,
"write",
project_to_groups.c.write,
"delete",
project_to_groups.c.delete,
),
).label("access_rights"),
)
sa.select(project_to_groups.c.project_uuid)
.where(
(
project_to_groups.c.read
Expand All @@ -427,7 +391,6 @@ def _create_private_workspace_query(
sa.select(
*PROJECT_DB_COLS,
projects.c.workbench,
access_rights_subquery.c.access_rights,
projects_to_products.c.product_name,
projects_to_folders.c.folder_id,
)
Expand All @@ -442,10 +405,6 @@ def _create_private_workspace_query(
),
isouter=True,
)
.join(
access_rights_subquery,
access_rights_subquery.c.project_uuid == projects.c.uuid,
)
)
.where(
(projects.c.workspace_id.is_(None)) # <-- Private workspace
Expand Down Expand Up @@ -479,46 +438,8 @@ def _create_shared_workspace_query(
WorkspaceScope.ALL,
)

workspace_access_rights_subquery = (
(
sa.select(
workspaces_access_rights.c.workspace_id,
sa.func.jsonb_object_agg(
workspaces_access_rights.c.gid,
sa.func.jsonb_build_object(
"read",
workspaces_access_rights.c.read,
"write",
workspaces_access_rights.c.write,
"delete",
workspaces_access_rights.c.delete,
),
).label("access_rights"),
)
.where(
workspaces_access_rights.c.read,
)
.group_by(workspaces_access_rights.c.workspace_id)
)
.subquery("workspace_access_rights_subquery")
.lateral()
)

my_workspace_access_rights_subquery = (
sa.select(
workspaces_access_rights.c.workspace_id,
sa.func.jsonb_object_agg(
workspaces_access_rights.c.gid,
sa.func.jsonb_build_object(
"read",
workspaces_access_rights.c.read,
"write",
workspaces_access_rights.c.write,
"delete",
workspaces_access_rights.c.delete,
),
).label("access_rights"),
)
sa.select(workspaces_access_rights.c.workspace_id)
.where(
workspaces_access_rights.c.read,
workspaces_access_rights.c.gid.in_(user_groups),
Expand All @@ -530,7 +451,6 @@ def _create_shared_workspace_query(
sa.select(
*PROJECT_DB_COLS,
projects.c.workbench,
workspace_access_rights_subquery.c.access_rights,
projects_to_products.c.product_name,
projects_to_folders.c.folder_id,
)
Expand All @@ -549,11 +469,6 @@ def _create_shared_workspace_query(
),
isouter=True,
)
.join(
workspace_access_rights_subquery,
projects.c.workspace_id
== workspace_access_rights_subquery.c.workspace_id,
)
)
.where(projects_to_products.c.product_name == product_name)
)
Expand Down
Loading