🎨 improve project full search (ITISFoundation#6483)

Author: matusdrobuliak66

api/specs/web-server/_projects_crud.py

@@ -142,10 +142,17 @@ async def clone_project(
 
 @router.get(
     "/projects:search",
-    response_model=Page[ProjectListItem],
+    response_model=Page[ProjectListFullSearchParams],
 )
 async def list_projects_full_search(
     _params: Annotated[ProjectListFullSearchParams, Depends()],
+    order_by: Annotated[
+        Json,
+        Query(
+            description="Order by field (type|uuid|name|description|prj_owner|creation_date|last_change_date) and direction (asc|desc). The default sorting order is ascending.",
+            example='{"field": "last_change_date", "direction": "desc"}',
+        ),
+    ] = ('{"field": "last_change_date", "direction": "desc"}',),
 ):
     ...
 

packages/service-library/src/servicelib/aiohttp/requests_validation.py

@@ -166,7 +166,10 @@ def parse_request_query_parameters_as(
         resource_name=request.rel_url.path,
         use_error_v1=use_enveloped_error_v1,
     ):
+        # NOTE: Currently, this does not take into consideration cases where there are multiple
+        # query parameters with the same key. However, we are not using such cases anywhere at the moment.
         data = dict(request.query)
+
         if hasattr(parameters_schema_cls, "parse_obj"):
             return parameters_schema_cls.parse_obj(data)
         model: ModelClass = parse_obj_as(parameters_schema_cls, data)

services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml

@@ -3298,6 +3298,18 @@ paths:
       summary: List Projects Full Search
       operationId: list_projects_full_search
       parameters:
+      - description: Order by field (type|uuid|name|description|prj_owner|creation_date|last_change_date)
+          and direction (asc|desc). The default sorting order is ascending.
+        required: false
+        schema:
+          title: Order By
+          description: Order by field (type|uuid|name|description|prj_owner|creation_date|last_change_date)
+            and direction (asc|desc). The default sorting order is ascending.
+          default:
+          - '{"field": "last_change_date", "direction": "desc"}'
+        example: '{"field": "last_change_date", "direction": "desc"}'
+        name: order_by
+        in: query
       - required: false
         schema:
           title: Limit
@@ -3323,13 +3335,19 @@ paths:
           type: string
         name: text
         in: query
+      - required: false
+        schema:
+          title: Tag Ids
+          type: string
+        name: tag_ids
+        in: query
       responses:
         '200':
           description: Successful Response
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/Page_ProjectListItem_'
+                $ref: '#/components/schemas/Page_ProjectListFullSearchParams_'
   /v0/projects/{project_id}/inactivity:
     get:
       tags:
@@ -9555,6 +9573,25 @@ components:
             $ref: '#/components/schemas/ProjectIterationResultItem'
       additionalProperties: false
       description: Paginated response model of ItemTs
+    Page_ProjectListFullSearchParams_:
+      title: Page[ProjectListFullSearchParams]
+      required:
+      - _meta
+      - _links
+      - data
+      type: object
+      properties:
+        _meta:
+          $ref: '#/components/schemas/PageMetaInfoLimitOffset'
+        _links:
+          $ref: '#/components/schemas/PageLinks'
+        data:
+          title: Data
+          type: array
+          items:
+            $ref: '#/components/schemas/ProjectListFullSearchParams'
+      additionalProperties: false
+      description: Paginated response model of ItemTs
     Page_ProjectListItem_:
       title: Page[ProjectListItem]
       required:
@@ -10414,6 +10451,37 @@ components:
           format: uri
         results:
           $ref: '#/components/schemas/ExtractedResults'
+    ProjectListFullSearchParams:
+      title: ProjectListFullSearchParams
+      type: object
+      properties:
+        limit:
+          title: Limit
+          exclusiveMaximum: true
+          minimum: 1
+          type: integer
+          description: maximum number of items to return (pagination)
+          default: 20
+          maximum: 50
+        offset:
+          title: Offset
+          minimum: 0
+          type: integer
+          description: index to the first item to return (pagination)
+          default: 0
+        text:
+          title: Text
+          maxLength: 100
+          type: string
+          description: Multi column full text search, across all folders and workspaces
+          example: My Project
+        tag_ids:
+          title: Tag Ids
+          type: string
+          description: Search by tag ID (multiple tag IDs may be provided separated
+            by column)
+          example: 1,3
+      description: Use as pagination options in query parameters
     ProjectListItem:
       title: ProjectListItem
       required:

services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py

@@ -143,25 +143,54 @@ async def list_projects(  # pylint: disable=too-many-arguments
 
 
 async def list_projects_full_search(
-    app,
+    request,
     *,
     user_id: UserID,
     product_name: str,
     offset: NonNegativeInt,
     limit: int,
     text: str | None,
+    order_by: OrderBy,
+    tag_ids_list: list[int],
 ) -> tuple[list[ProjectDict], int]:
-    db = ProjectDBAPI.get_from_app_context(app)
+    db = ProjectDBAPI.get_from_app_context(request.app)
+
+    user_available_services: list[dict] = await get_services_for_user_in_product(
+        request.app, user_id, product_name, only_key_versions=True
+    )
 
-    total_number_projects, db_projects = await db.list_projects_full_search(
+    (
+        db_projects,
+        db_project_types,
+        total_number_projects,
+    ) = await db.list_projects_full_search(
         user_id=user_id,
         product_name=product_name,
+        filter_by_services=user_available_services,
         text=text,
         offset=offset,
         limit=limit,
+        order_by=order_by,
+        tag_ids_list=tag_ids_list,
     )
 
-    return db_projects, total_number_projects
+    projects: list[ProjectDict] = await logged_gather(
+        *(
+            _append_fields(
+                request,
+                user_id=user_id,
+                project=prj,
+                is_template=prj_type == ProjectTypeDB.TEMPLATE,
+                workspace_access_rights=None,
+                model_schema_cls=ProjectListItem,
+            )
+            for prj, prj_type in zip(db_projects, db_project_types)
+        ),
+        reraise=True,
+        max_concurrency=100,
+    )
+
+    return projects, total_number_projects
 
 
 async def get_project(

services/web/server/src/simcore_service_webserver/projects/_crud_handlers.py

@@ -58,7 +58,7 @@
     ProjectActiveParams,
     ProjectCreateHeaders,
     ProjectCreateParams,
-    ProjectListFullSearchParams,
+    ProjectListFullSearchWithJsonStrParams,
     ProjectListWithJsonStrParams,
 )
 from ._permalink_api import update_or_pop_permalink_in_project
@@ -69,6 +69,7 @@
     ProjectInvalidUsageError,
     ProjectNotFoundError,
     ProjectOwnerNotFoundInTheProjectAccessRightsError,
+    WrongTagIdsInQueryError,
 )
 from .lock import get_project_locked_state
 from .models import ProjectDict
@@ -101,7 +102,10 @@ async def _wrapper(request: web.Request) -> web.StreamResponse:
             WorkspaceNotFoundError,
         ) as exc:
             raise web.HTTPNotFound(reason=f"{exc}") from exc
-        except ProjectOwnerNotFoundInTheProjectAccessRightsError as exc:
+        except (
+            ProjectOwnerNotFoundInTheProjectAccessRightsError,
+            WrongTagIdsInQueryError,
+        ) as exc:
             raise web.HTTPBadRequest(reason=f"{exc}") from exc
         except (
             ProjectInvalidRightsError,
@@ -233,17 +237,22 @@ async def list_projects(request: web.Request):
 @_handle_projects_exceptions
 async def list_projects_full_search(request: web.Request):
     req_ctx = RequestContext.parse_obj(request)
-    query_params: ProjectListFullSearchParams = parse_request_query_parameters_as(
-        ProjectListFullSearchParams, request
+    query_params: ProjectListFullSearchWithJsonStrParams = (
+        parse_request_query_parameters_as(
+            ProjectListFullSearchWithJsonStrParams, request
+        )
     )
+    tag_ids_list = query_params.tag_ids_list()
 
     projects, total_number_of_projects = await _crud_api_read.list_projects_full_search(
-        request.app,
+        request,
         user_id=req_ctx.user_id,
         product_name=req_ctx.product_name,
         limit=query_params.limit,
         offset=query_params.offset,
         text=query_params.text,
+        order_by=query_params.order_by,
+        tag_ids_list=tag_ids_list,
     )
 
     page = Page[ProjectDict].parse_obj(

services/web/server/src/simcore_service_webserver/projects/_crud_handlers_models.py

@@ -17,14 +17,23 @@
     null_or_none_str_to_none_validator,
 )
 from models_library.workspaces import WorkspaceID
-from pydantic import BaseModel, Extra, Field, Json, root_validator, validator
+from pydantic import (
+    BaseModel,
+    Extra,
+    Field,
+    Json,
+    parse_obj_as,
+    root_validator,
+    validator,
+)
 from servicelib.common_headers import (
     UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE,
     X_SIMCORE_PARENT_NODE_ID,
     X_SIMCORE_PARENT_PROJECT_UUID,
     X_SIMCORE_USER_AGENT,
 )
 
+from .exceptions import WrongTagIdsInQueryError
 from .models import ProjectTypeAPI
 
 
@@ -123,7 +132,7 @@ def search_check_empty_string(cls, v):
     )(null_or_none_str_to_none_validator)
 
 
-class ProjectListWithJsonStrParams(ProjectListParams):
+class ProjectListWithOrderByParams(BaseModel):
     order_by: Json[OrderBy] = Field(  # pylint: disable=unsubscriptable-object
         default=OrderBy(field=IDStr("last_change_date"), direction=OrderDirection.DESC),
         description="Order by field (type|uuid|name|description|prj_owner|creation_date|last_change_date) and direction (asc|desc). The default sorting order is ascending.",
@@ -151,6 +160,10 @@ class Config:
         extra = Extra.forbid
 
 
+class ProjectListWithJsonStrParams(ProjectListParams, ProjectListWithOrderByParams):
+    ...
+
+
 class ProjectActiveParams(BaseModel):
     client_session_id: str
 
@@ -162,7 +175,30 @@ class ProjectListFullSearchParams(PageQueryParameters):
         max_length=100,
         example="My Project",
     )
+    tag_ids: str | None = Field(
+        default=None,
+        description="Search by tag ID (multiple tag IDs may be provided separated by column)",
+        example="1,3",
+    )
 
     _empty_is_none = validator("text", allow_reuse=True, pre=True)(
         empty_str_to_none_pre_validator
     )
+
+
+class ProjectListFullSearchWithJsonStrParams(
+    ProjectListFullSearchParams, ProjectListWithOrderByParams
+):
+    def tag_ids_list(self) -> list[int]:
+        try:
+            # Split the tag_ids by commas and map them to integers
+            if self.tag_ids:
+                tag_ids_list = list(map(int, self.tag_ids.split(",")))
+                # Validate that the tag_ids_list is indeed a list of integers
+                parse_obj_as(list[int], tag_ids_list)
+            else:
+                tag_ids_list = []
+        except ValueError as exc:
+            raise WrongTagIdsInQueryError from exc
+
+        return tag_ids_list

services/web/server/src/simcore_service_webserver/projects/_db_utils.py

@@ -20,7 +20,7 @@
 from simcore_postgres_database.webserver_models import ProjectType, projects
 from sqlalchemy.dialects.postgresql import insert as pg_insert
 from sqlalchemy.sql import select
-from sqlalchemy.sql.selectable import Select
+from sqlalchemy.sql.selectable import CompoundSelect, Select
 
 from ..db.models import GroupType, groups, projects_tags, user_to_groups, users
 from ..users.exceptions import UserNotFoundError
@@ -181,7 +181,7 @@ async def _execute_without_permission_check(
         conn: SAConnection,
         user_id: UserID,
         *,
-        select_projects_query: Select,
+        select_projects_query: Select | CompoundSelect,
         filter_by_services: list[dict] | None = None,
     ) -> tuple[list[dict[str, Any]], list[ProjectType]]:
         api_projects: list[dict] = []  # API model-compatible projects