Merge remote-tracking branch 'upstream/master' into pr-osparc-long-running-rabbitmq-client

Author: Andrei Neagu

.env-devel

@@ -195,7 +195,8 @@ POSTGRES_HOST=postgres
 POSTGRES_PASSWORD=adminadmin
 POSTGRES_PORT=5432
 POSTGRES_USER=scu
-
+POSTGRES_MINSIZE=2 # see https://github.com/ITISFoundation/osparc-simcore/pull/8199
+POSTGRES_MAXSIZE=50
 POSTGRES_READONLY_PASSWORD=readonly
 POSTGRES_READONLY_USER=postgres_readonly
 

packages/models-library/src/models_library/api_schemas_webserver/projects.py

@@ -278,6 +278,7 @@ class ProjectPatch(InputSchema):
     ] = None
     quality: dict[str, Any] | None = None
     template_type: ProjectTemplateType | None = None
+    hidden: bool | None = None
 
     def to_domain_model(self) -> dict[str, Any]:
         return self.model_dump(exclude_unset=True, by_alias=False)

packages/models-library/src/models_library/rpc/webserver/projects.py

@@ -90,6 +90,7 @@ class ProjectJobRpcGet(BaseModel):
 
     # Specific to jobs
     job_parent_resource_name: str
+    storage_assets_deleted: bool
 
     @staticmethod
     def _update_json_schema_extra(schema: JsonDict) -> None:
@@ -105,6 +106,7 @@ def _update_json_schema_extra(schema: JsonDict) -> None:
                         "created_at": "2023-01-01T00:00:00Z",
                         "modified_at": "2023-01-01T00:00:00Z",
                         "job_parent_resource_name": "solvers/simcore%2Fservices%2Fcomp%2Fitis%2Fsleeper/releases/2.0.2",
+                        "storage_assets_deleted": "false",
                     },
                     {
                         "uuid": "00000000-1234-5678-1234-123456789012",
@@ -114,6 +116,7 @@ def _update_json_schema_extra(schema: JsonDict) -> None:
                         "created_at": "2023-02-01T00:00:00Z",
                         "modified_at": "2023-02-01T00:00:00Z",
                         "job_parent_resource_name": "studies/96642f2a-a72c-11ef-8776-02420a00087d",
+                        "storage_assets_deleted": "true",
                     },
                     {
                         "uuid": "00000000-0000-5678-1234-123456789012",
@@ -123,6 +126,7 @@ def _update_json_schema_extra(schema: JsonDict) -> None:
                         "created_at": "2023-03-01T00:00:00Z",
                         "modified_at": "2023-03-01T00:00:00Z",
                         "job_parent_resource_name": "program/simcore%2Fservices%2Fdynamic%2Fjupyter/releases/5.0.2",
+                        "storage_assets_deleted": "false",
                     },
                 ]
             }

packages/postgres-database/src/simcore_postgres_database/migration/versions/5b998370916a_introduce_data_deleted_in_projects_to_.py

@@ -0,0 +1,40 @@
+"""introduce data_deleted in projects_to_jobs table
+
+Revision ID: 5b998370916a
+Revises: 5679165336c8
+Create Date: 2025-08-11 13:58:38.424398+00:00
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "5b998370916a"
+down_revision = "5679165336c8"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column(
+        "projects_to_jobs",
+        sa.Column("storage_assets_deleted", sa.Boolean(), nullable=True),
+    )
+
+    op.execute("UPDATE projects_to_jobs SET storage_assets_deleted = false")
+
+    op.alter_column(
+        "projects_to_jobs",
+        "storage_assets_deleted",
+        existing_type=sa.BOOLEAN(),
+        nullable=False,
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("projects_to_jobs", "storage_assets_deleted")
+    # ### end Alembic commands ###

packages/postgres-database/src/simcore_postgres_database/models/projects_to_jobs.py

@@ -28,6 +28,12 @@
         "the relative resource name is shelves/shelf1/jobs/job2, "
         "the parent resource name is shelves/shelf1.",
     ),
+    sa.Column(
+        "storage_assets_deleted",
+        sa.Boolean,
+        nullable=False,
+        doc="Indicates whether the job's S3 assets have been actively deleted.",
+    ),
     # Composite key (project_uuid, job_parent_resource_name) uniquely identifies very row
     sa.UniqueConstraint(
         "project_uuid",

packages/postgres-database/tests/conftest.py

@@ -81,7 +81,7 @@ def sync_engine(postgres_service: str) -> Iterable[sqlalchemy.engine.Engine]:
 def _make_asyncpg_engine(postgres_service: str) -> Callable[[bool], AsyncEngine]:
     # NOTE: users is responsible of `await engine.dispose()`
     dsn = postgres_service.replace("postgresql://", "postgresql+asyncpg://")
-    minsize = 1
+    minsize = 2
     maxsize = 50
 
     def _(echo: bool):

packages/pytest-simcore/src/pytest_simcore/helpers/webserver_rpc_server.py

@@ -26,6 +26,14 @@
 class WebserverRpcSideEffects:
     # pylint: disable=no-self-use
 
+    def __init__(
+        self,
+        project_job_rpc_get: ProjectJobRpcGet = ProjectJobRpcGet.model_validate(
+            ProjectJobRpcGet.model_json_schema()["examples"][0]
+        ),
+    ):
+        self.project_job_rpc_get = project_job_rpc_get
+
     @validate_call(config={"arbitrary_types_allowed": True})
     async def mark_project_as_job(
         self,
@@ -35,12 +43,14 @@ async def mark_project_as_job(
         user_id: UserID,
         project_uuid: ProjectID,
         job_parent_resource_name: str,
+        storage_assets_deleted: bool,
     ) -> None:
         assert rpc_client
 
         assert not job_parent_resource_name.startswith("/")  # nosec
         assert "/" in job_parent_resource_name  # nosec
         assert not job_parent_resource_name.endswith("/")  # nosec
+        assert isinstance(storage_assets_deleted, bool)
 
         assert product_name
         assert user_id
@@ -84,3 +94,25 @@ async def list_projects_marked_as_jobs(
             limit=limit,
             offset=offset,
         )
+
+    @validate_call(config={"arbitrary_types_allowed": True})
+    async def get_project_marked_as_job(
+        self,
+        rpc_client: RabbitMQRPCClient | MockType,
+        *,
+        product_name: ProductName,
+        user_id: UserID,
+        project_uuid: ProjectID,
+        job_parent_resource_name: str,
+    ) -> ProjectJobRpcGet:
+        assert rpc_client
+        assert product_name
+        assert user_id
+        assert project_uuid
+        assert job_parent_resource_name
+
+        # Return a valid example from the schema
+        _data = self.project_job_rpc_get.model_dump()
+        _data["uuid"] = str(project_uuid)
+        _data["job_parent_resource_name"] = job_parent_resource_name
+        return ProjectJobRpcGet.model_validate(_data)

packages/service-library/src/servicelib/db_asyncpg_utils.py

@@ -30,12 +30,16 @@ async def create_async_engine_and_database_ready(
         raise_if_migration_not_ready,
     )
 
-    server_settings = None
+    server_settings = {
+        "jit": "off"
+    }  # see https://docs.sqlalchemy.org/en/20/dialects/postgresql.html#disabling-the-postgresql-jit-to-improve-enum-datatype-handling
     if settings.POSTGRES_CLIENT_NAME:
         assert isinstance(settings.POSTGRES_CLIENT_NAME, str)  # nosec
-        server_settings = {
-            "application_name": settings.POSTGRES_CLIENT_NAME,
-        }
+        server_settings.update(
+            {
+                "application_name": settings.POSTGRES_CLIENT_NAME,
+            }
+        )
 
     engine = create_async_engine(
         settings.dsn_with_async_sqlalchemy,

packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py

@@ -17,13 +17,15 @@
     RegisteredFunctionJobCollection,
 )
 from models_library.functions import (
+    FunctionClass,
     FunctionJobStatus,
     FunctionOutputs,
     FunctionUserAccessRights,
     FunctionUserApiAccessRights,
 )
 from models_library.products import ProductName
 from models_library.rabbitmq_basic_types import RPCMethodName
+from models_library.rest_ordering import OrderBy
 from models_library.rest_pagination import PageMetaInfoLimitOffset
 from models_library.users import UserID
 from pydantic import TypeAdapter
@@ -135,6 +137,10 @@ async def list_functions(
     product_name: ProductName,
     pagination_offset: int,
     pagination_limit: int,
+    order_by: OrderBy | None = None,
+    filter_by_function_class: FunctionClass | None = None,
+    search_by_function_title: str | None = None,
+    search_by_multi_columns: str | None = None,
 ) -> tuple[list[RegisteredFunction], PageMetaInfoLimitOffset]:
     result: tuple[list[RegisteredFunction], PageMetaInfoLimitOffset] = (
         await rabbitmq_rpc_client.request(
@@ -144,6 +150,10 @@ async def list_functions(
             pagination_limit=pagination_limit,
             user_id=user_id,
             product_name=product_name,
+            order_by=order_by,
+            filter_by_function_class=filter_by_function_class,
+            search_by_function_title=search_by_function_title,
+            search_by_multi_columns=search_by_multi_columns,
         )
     )
     return TypeAdapter(

packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/projects.py

@@ -9,6 +9,7 @@
 from models_library.rpc.webserver.projects import (
     ListProjectsMarkedAsJobRpcFilters,
     PageRpcProjectJobRpcGet,
+    ProjectJobRpcGet,
 )
 from models_library.rpc_pagination import (
     DEFAULT_NUMBER_OF_ITEMS_PER_PAGE,
@@ -32,6 +33,7 @@ async def mark_project_as_job(
     user_id: UserID,
     project_uuid: ProjectID,
     job_parent_resource_name: str,
+    storage_assets_deleted: bool,
 ) -> None:
 
     result = await rpc_client.request(
@@ -41,6 +43,7 @@ async def mark_project_as_job(
         user_id=user_id,
         project_uuid=project_uuid,
         job_parent_resource_name=job_parent_resource_name,
+        storage_assets_deleted=storage_assets_deleted,
     )
     assert result is None
 
@@ -68,3 +71,25 @@ async def list_projects_marked_as_jobs(
     )
     assert TypeAdapter(PageRpcProjectJobRpcGet).validate_python(result)  # nosec
     return cast(PageRpcProjectJobRpcGet, result)
+
+
+@log_decorator(_logger, level=logging.DEBUG)
+@validate_call(config={"arbitrary_types_allowed": True})
+async def get_project_marked_as_job(
+    rpc_client: RabbitMQRPCClient,
+    *,
+    product_name: ProductName,
+    user_id: UserID,
+    project_uuid: ProjectID,
+    job_parent_resource_name: str,
+) -> ProjectJobRpcGet:
+    result = await rpc_client.request(
+        WEBSERVER_RPC_NAMESPACE,
+        TypeAdapter(RPCMethodName).validate_python("get_project_marked_as_job"),
+        product_name=product_name,
+        user_id=user_id,
+        project_uuid=project_uuid,
+        job_parent_resource_name=job_parent_resource_name,
+    )
+    assert TypeAdapter(ProjectJobRpcGet).validate_python(result)  # nosec
+    return cast(ProjectJobRpcGet, result)