🎨 make folders optional + improvements ⚠️ (#6155)

Author: matusdrobuliak66

.env-devel

@@ -318,6 +318,7 @@ WEBSERVER_DIAGNOSTICS={}
 WEBSERVER_EMAIL={}
 WEBSERVER_EXPORTER={}
 WEBSERVER_FRONTEND={}
+WEBSERVER_FOLDERS=1
 WEBSERVER_GARBAGE_COLLECTOR=null
 WEBSERVER_GROUPS=1
 WEBSERVER_GUNICORN_CMD_ARGS=--timeout=180

api/specs/web-server/_folders.py

@@ -15,7 +15,6 @@
     FolderGet,
     PutFolderBodyParams,
 )
-from models_library.api_schemas_webserver.wallets import WalletGet
 from models_library.generics import Envelope
 from models_library.rest_pagination import PageQueryParameters
 from pydantic import Json
@@ -39,7 +38,7 @@
 
 @router.post(
     "/folders",
-    response_model=Envelope[WalletGet],
+    response_model=Envelope[FolderGet],
     status_code=status.HTTP_201_CREATED,
 )
 async def create_folder(_body: CreateFolderBodyParams):
@@ -55,10 +54,10 @@ async def list_folders(
     order_by: Annotated[
         Json,
         Query(
-            description="Order by field (name|description) and direction (asc|desc). The default sorting order is ascending.",
+            description="Order by field (modified_at|name|description) and direction (asc|desc). The default sorting order is ascending.",
             example='{"field": "name", "direction": "desc"}',
         ),
-    ] = '{"field": "name", "direction": "desc"}',
+    ] = '{"field": "modified_at", "direction": "desc"}',
 ):
     ...
 
@@ -83,7 +82,7 @@ async def replace_folder(
 
 @router.delete(
     "/folders/{folder_id}",
-    response_model=Envelope[FolderGet],
+    status_code=status.HTTP_204_NO_CONTENT,
 )
 async def delete_folder(_path: Annotated[FoldersPathParams, Depends()]):
     ...

packages/models-library/src/models_library/api_schemas_webserver/folders.py

@@ -1,10 +1,12 @@
 from datetime import datetime
+from typing import NamedTuple
 
 from models_library.basic_types import IDStr
 from models_library.folders import FolderID
 from models_library.projects_access import AccessRights
 from models_library.users import GroupID
-from pydantic import Extra
+from models_library.utils.common_validators import null_or_none_str_to_none_validator
+from pydantic import Extra, PositiveInt, validator
 
 from ._base import InputSchema, OutputSchema
 
@@ -21,6 +23,11 @@ class FolderGet(OutputSchema):
     access_rights: dict[GroupID, AccessRights]
 
 
+class FolderGetPage(NamedTuple):
+    items: list[FolderGet]
+    total: PositiveInt
+
+
 class CreateFolderBodyParams(InputSchema):
     name: IDStr
     description: str
@@ -29,6 +36,10 @@ class CreateFolderBodyParams(InputSchema):
     class Config:
         extra = Extra.forbid
 
+    _null_or_none_str_to_none_validator = validator(
+        "parent_folder_id", allow_reuse=True, pre=True
+    )(null_or_none_str_to_none_validator)
+
 
 class PutFolderBodyParams(InputSchema):
     name: IDStr

packages/models-library/src/models_library/utils/common_validators.py

@@ -63,3 +63,9 @@ def ensure_unique_dict_values_validator(dict_data: dict) -> dict:
         msg = f"Dictionary values must be unique, provided: {dict_data}"
         raise ValueError(msg)
     return dict_data
+
+
+def null_or_none_str_to_none_validator(value: Any):
+    if isinstance(value, str) and value.lower() in ("null", "none"):
+        return None
+    return value

packages/models-library/tests/test_utils_common_validators.py

@@ -5,6 +5,7 @@
     create_enums_pre_validator,
     empty_str_to_none_pre_validator,
     none_to_empty_str_pre_validator,
+    null_or_none_str_to_none_validator,
 )
 from pydantic import BaseModel, ValidationError, validator
 
@@ -59,3 +60,30 @@ class Model(BaseModel):
 
     model = Model.parse_obj({"message": ""})
     assert model == Model.parse_obj({"message": None})
+
+
+def test_null_or_none_str_to_none_validator():
+    class Model(BaseModel):
+        message: str | None
+
+        _null_or_none_str_to_none_validator = validator(
+            "message", allow_reuse=True, pre=True
+        )(null_or_none_str_to_none_validator)
+
+    model = Model.parse_obj({"message": "none"})
+    assert model == Model.parse_obj({"message": None})
+
+    model = Model.parse_obj({"message": "null"})
+    assert model == Model.parse_obj({"message": None})
+
+    model = Model.parse_obj({"message": "NoNe"})
+    assert model == Model.parse_obj({"message": None})
+
+    model = Model.parse_obj({"message": "NuLl"})
+    assert model == Model.parse_obj({"message": None})
+
+    model = Model.parse_obj({"message": None})
+    assert model == Model.parse_obj({"message": None})
+
+    model = Model.parse_obj({"message": ""})
+    assert model == Model.parse_obj({"message": ""})

packages/postgres-database/src/simcore_postgres_database/utils_folders.py

@@ -5,7 +5,7 @@
 from datetime import datetime
 from enum import Enum
 from functools import reduce
-from typing import Any, ClassVar, Final, TypeAlias
+from typing import Any, ClassVar, Final, TypeAlias, cast
 
 import sqlalchemy as sa
 from aiopg.sa.connection import SAConnection
@@ -20,12 +20,14 @@
     parse_obj_as,
 )
 from pydantic.errors import PydanticErrorMixin
+from simcore_postgres_database.utils_ordering import OrderByDict
 from sqlalchemy.dialects import postgresql
 from sqlalchemy.sql.elements import ColumnElement
 from sqlalchemy.sql.selectable import ScalarSelect
 
 from .models.folders import folders, folders_access_rights, folders_to_projects
 from .models.groups import GroupType, groups
+from .utils_ordering import OrderDirection
 
 _ProductName: TypeAlias = str
 _ProjectID: TypeAlias = uuid.UUID
@@ -986,8 +988,11 @@ async def folder_list(
     *,
     offset: NonNegativeInt,
     limit: NonNegativeInt,
+    order_by: OrderByDict = OrderByDict(
+        field="modified", direction=OrderDirection.DESC
+    ),
     _required_permissions=_requires(_BasePermissions.LIST_FOLDERS),  # noqa: B008
-) -> list[FolderEntry]:
+) -> tuple[int, list[FolderEntry]]:
     """
     Raises:
         FolderNotFoundError
@@ -1015,7 +1020,7 @@ async def folder_list(
             access_via_gid = resolved_access_rights.gid
             access_via_folder_id = resolved_access_rights.folder_id
 
-        query = (
+        base_query = (
             sa.select(
                 folders,
                 folders_access_rights,
@@ -1047,14 +1052,30 @@ async def folder_list(
                 if folder_id is None
                 else True
             )
-            .offset(offset)
-            .limit(limit)
+            .where(folders.c.product_name == product_name)
         )
 
-        async for entry in connection.execute(query):
+        # Select total count from base_query
+        subquery = base_query.subquery()
+        count_query = sa.select(sa.func.count()).select_from(subquery)
+        count_result = await connection.execute(count_query)
+        total_count = await count_result.scalar()
+
+        # Ordering and pagination
+        if order_by["direction"] == OrderDirection.ASC:
+            list_query = base_query.order_by(
+                sa.asc(getattr(folders.c, order_by["field"]))
+            )
+        else:
+            list_query = base_query.order_by(
+                sa.desc(getattr(folders.c, order_by["field"]))
+            )
+        list_query = list_query.offset(offset).limit(limit)
+
+        async for entry in connection.execute(list_query):
             results.append(FolderEntry.from_orm(entry))  # noqa: PERF401s
 
-    return results
+    return cast(int, total_count), results
 
 
 async def folder_get(
@@ -1101,6 +1122,7 @@ async def folder_get(
                 if folder_id is None
                 else True
             )
+            .where(folders.c.product_name == product_name)
         )
 
         query_result: RowProxy | None = await (
@@ -1113,3 +1135,6 @@ async def folder_get(
         )
 
     return FolderEntry.from_orm(query_result)
+
+
+__all__ = ["OrderByDict"]

packages/postgres-database/src/simcore_postgres_database/utils_ordering.py

@@ -0,0 +1,19 @@
+from enum import Enum
+from typing import TypedDict
+
+
+class OrderDirection(str, Enum):
+    ASC = "asc"
+    DESC = "desc"
+
+
+class OrderByDict(TypedDict):
+    field: str
+    direction: OrderDirection
+
+
+# Example usage
+order_by_example: OrderByDict = {
+    "field": "example_field",
+    "direction": OrderDirection.ASC,
+}

packages/postgres-database/tests/test_utils_folders.py

@@ -1633,9 +1633,10 @@ async def _list_folder_as(
     limit: NonNegativeInt = ALL_IN_ONE_PAGE_LIMIT,
 ) -> list[FolderEntry]:
 
-    return await folder_list(
+    total_count, folders_db = await folder_list(
         connection, default_product_name, folder_id, gid, offset=offset, limit=limit
     )
+    return folders_db
 
 
 async def test_folder_list(

services/docker-compose.yml

@@ -725,6 +725,7 @@ services:
       WEBSERVER_TAGS: ${WEBSERVER_TAGS}
       WEBSERVER_USERS: ${WEBSERVER_USERS}
       WEBSERVER_VERSION_CONTROL: ${WEBSERVER_VERSION_CONTROL}
+      WEBSERVER_FOLDERS: ${WEBSERVER_FOLDERS}
 
     deploy:
       labels:

services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml

@@ -1296,14 +1296,14 @@ paths:
       summary: List Folders
       operationId: list_folders
       parameters:
-      - description: Order by field (name|description) and direction (asc|desc). The
-          default sorting order is ascending.
+      - description: Order by field (modified_at|name|description) and direction (asc|desc).
+          The default sorting order is ascending.
         required: false
         schema:
           title: Order By
-          description: Order by field (name|description) and direction (asc|desc).
-            The default sorting order is ascending.
-          default: '{"field": "name", "direction": "desc"}'
+          description: Order by field (modified_at|name|description) and direction
+            (asc|desc). The default sorting order is ascending.
+          default: '{"field": "modified_at", "direction": "desc"}'
         example: '{"field": "name", "direction": "desc"}'
         name: order_by
         in: query
@@ -1349,7 +1349,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/Envelope_WalletGet_'
+                $ref: '#/components/schemas/Envelope_FolderGet_'
   /v0/folders/{folder_id}:
     get:
       tags:
@@ -1414,12 +1414,8 @@ paths:
         name: folder_id
         in: path
       responses:
-        '200':
+        '204':
           description: Successful Response
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/Envelope_FolderGet_'
   /v0/folders/{folder_id}/groups/{group_id}:
     put:
       tags:
@@ -6008,8 +6004,8 @@ components:
         quality: {}
         accessRights:
           '1':
-            execute_access: true
-            write_access: false
+            execute: true
+            write: false
         key: simcore/services/comp/itis/sleeper
         version: 2.2.1
         version_display: 2 Xtreme
@@ -10788,6 +10784,7 @@ components:
           title: Write
           type: boolean
           default: false
+      additionalProperties: false
     ServiceInputGet:
       title: ServiceInputGet
       required: