drafting filter and tests

Author: pcrespov

packages/models-library/src/models_library/projects_filters.py

@@ -0,0 +1,18 @@
+from typing import Annotated
+
+from pydantic import BaseModel, Field
+
+from .basic_types import IDStr
+
+
+class MetadataFilterItem(BaseModel):
+    key: IDStr
+    value: Annotated[str | None, Field(description="SQL-like pattern")]
+
+
+class ListProjectsMarkedAsJobFilter(BaseModel):
+    job_parent_resource_name_prefix: str | None
+    any_of_metadata: list[MetadataFilterItem] | None = None
+
+    # TODO: early validation of filters
+    # TODO: update interface to list_projects_marked_as_jobs

packages/models-library/src/models_library/rpc/webserver/projects.py

@@ -2,12 +2,17 @@
 from typing import Annotated, TypeAlias
 from uuid import uuid4
 
-from models_library.projects import NodesDict, ProjectID
-from models_library.projects_nodes import Node
-from models_library.rpc_pagination import PageRpc
 from pydantic import BaseModel, ConfigDict, Field
 from pydantic.config import JsonDict
 
+from ...projects import NodesDict, ProjectID
+from ...projects_filters import ListProjectsMarkedAsJobFilter
+from ...projects_nodes import Node
+from ...rpc_pagination import PageRpc
+
+# NOTE: for now these interfaces are identical
+ListProjectsMarkedAsJobRpcFilter: TypeAlias = ListProjectsMarkedAsJobFilter
+
 
 class ProjectJobRpcGet(BaseModel):
     """

services/web/server/src/simcore_service_webserver/projects/_controller/projects_rpc.py

@@ -4,6 +4,7 @@
 from models_library.projects import ProjectID
 from models_library.rest_pagination import PageLimitInt, PageOffsetInt
 from models_library.rpc.webserver.projects import (
+    ListProjectsMarkedAsJobRpcFilter,
     PageRpcProjectJobRpcGet,
     ProjectJobRpcGet,
 )
@@ -66,7 +67,9 @@ async def list_projects_marked_as_jobs(
     offset: PageOffsetInt,
     limit: PageLimitInt,
     # filters
+    # TODO: replace by ListProjectsMarkedAsJobsFilter
     job_parent_resource_name_prefix: str | None,
+    filters: ListProjectsMarkedAsJobRpcFilter | None = None,
 ) -> PageRpcProjectJobRpcGet:
 
     total, projects = await _jobs_service.list_my_projects_marked_as_jobs(
@@ -76,6 +79,7 @@ async def list_projects_marked_as_jobs(
         offset=offset,
         limit=limit,
         filter_by_job_parent_resource_name_prefix=job_parent_resource_name_prefix,
+        filter_by_metadata_any_of=filters.any_of_metadata if filters else None,
     )
 
     job_projects = [

services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py

@@ -8,6 +8,7 @@
 from simcore_postgres_database.models.groups import user_to_groups
 from simcore_postgres_database.models.project_to_groups import project_to_groups
 from simcore_postgres_database.models.projects import projects
+from simcore_postgres_database.models.projects_metadata import projects_metadata
 from simcore_postgres_database.models.projects_to_jobs import projects_to_jobs
 from simcore_postgres_database.models.projects_to_products import projects_to_products
 from simcore_postgres_database.utils_repos import (
@@ -30,6 +31,30 @@
 )
 
 
+def _apply_job_parent_resource_name_filter(
+    query: sa.sql.Select, prefix: str
+) -> sa.sql.Select:
+    return query.where(projects_to_jobs.c.job_parent_resource_name.like(f"{prefix}%"))
+
+
+def _apply_metadata_filter(
+    query: sa.sql.Select, any_of_metadata_fields: list[dict[str, str]]
+) -> sa.sql.Select:
+
+    assert any_of_metadata_fields  # nosec
+
+    expressions = [
+        projects_metadata.c.custom[key].ilike(
+            # NOTE: we use ilike to be case insensitive
+            # NOTE: supports glob wildcards
+            pattern.replace("*", "%")
+        )
+        for field in any_of_metadata_fields
+        for key, pattern in field.items()
+    ]
+    return query.where(sa.or_(*expressions))
+
+
 class ProjectJobsRepository(BaseRepository):
 
     async def set_project_as_job(
@@ -60,9 +85,10 @@ async def list_projects_marked_as_jobs(
         *,
         product_name: ProductName,
         user_id: UserID,
-        offset: int = 0,
-        limit: int = 10,
-        job_parent_resource_name_prefix: str | None = None,
+        pagination_offset: int = 0,
+        pagination_limit: int = 10,
+        filter_by_job_parent_resource_name_prefix: str | None = None,
+        filter_by_any_of_metadata_fields: list[dict[str, str]] | None = None,
     ) -> tuple[int, list[ProjectJobDBGet]]:
         """Lists projects marked as jobs for a specific user and product
 
@@ -96,10 +122,15 @@ async def list_projects_marked_as_jobs(
                     projects_to_products,
                     projects_to_jobs.c.project_uuid
                     == projects_to_products.c.project_uuid,
-                ).join(
+                )
+                .join(
                     project_to_groups,
                     projects_to_jobs.c.project_uuid == project_to_groups.c.project_uuid,
                 )
+                .outerjoin(
+                    projects_metadata,
+                    projects_to_jobs.c.project_uuid == projects_metadata.c.project_uuid,
+                )
             )
             .where(
                 projects_to_products.c.product_name == product_name,
@@ -112,21 +143,24 @@ async def list_projects_marked_as_jobs(
             )
         )
 
-        # Apply job_parent_resource_name_filter if provided
-        if job_parent_resource_name_prefix:
-            access_query = access_query.where(
-                projects_to_jobs.c.job_parent_resource_name.like(
-                    f"{job_parent_resource_name_prefix}%"
-                )
+        # Step 3: Apply filters
+        if filter_by_job_parent_resource_name_prefix:
+            access_query = _apply_job_parent_resource_name_filter(
+                access_query, filter_by_job_parent_resource_name_prefix
+            )
+
+        if filter_by_any_of_metadata_fields:
+            access_query = _apply_metadata_filter(
+                access_query, filter_by_any_of_metadata_fields
             )
 
-        # Convert access_query to a subquery
+        # Step 4. Convert access_query to a subquery
         base_query = access_query.subquery()
 
-        # Step 3: Query to get the total count
+        # Step 5: Query to get the total count
         total_query = sa.select(sa.func.count()).select_from(base_query)
 
-        # Step 4: Query to get the paginated list with full selection
+        # Step 6: Query to get the paginated list with full selection
         list_query = (
             sa.select(
                 *_PROJECT_DB_COLS,
@@ -143,8 +177,8 @@ async def list_projects_marked_as_jobs(
                 projects.c.creation_date.desc(),  # latests first
                 projects.c.id.desc(),
             )
-            .limit(limit)
-            .offset(offset)
+            .limit(pagination_limit)
+            .offset(pagination_offset)
         )
 
         # Step 5: Execute queries

services/web/server/src/simcore_service_webserver/projects/_jobs_service.py

@@ -57,6 +57,7 @@ async def list_my_projects_marked_as_jobs(
     offset: int = 0,
     limit: int = 10,
     filter_by_job_parent_resource_name_prefix: str | None = None,
+    filter_by_any_of_metadata: list[dict[str, str]] | None = None,
 ) -> tuple[int, list[ProjectJobDBGet]]:
     """
     Lists paginated projects marked as jobs for the given user and product.
@@ -66,7 +67,8 @@ async def list_my_projects_marked_as_jobs(
     return await repo.list_projects_marked_as_jobs(
         user_id=user_id,
         product_name=product_name,
-        offset=offset,
-        limit=limit,
-        job_parent_resource_name_prefix=filter_by_job_parent_resource_name_prefix,
+        pagination_offset=offset,
+        pagination_limit=limit,
+        filter_by_job_parent_resource_name_prefix=filter_by_job_parent_resource_name_prefix,
+        filter_by_any_of_metadata_fields=filter_by_any_of_metadata,
     )

services/web/server/tests/unit/with_dbs/02/test_projects__jobs_service.py

@@ -8,6 +8,7 @@
 import pytest
 from aiohttp.test_utils import TestClient
 from common_library.users_enums import UserRole
+from models_library.api_schemas_webserver.projects_metadata import MetadataDict
 from models_library.products import ProductName
 from models_library.projects import ProjectID
 from models_library.users import UserID
@@ -16,6 +17,9 @@
     list_my_projects_marked_as_jobs,
     set_project_as_job,
 )
+from simcore_service_webserver.projects._metadata_service import (
+    set_project_custom_metadata,
+)
 from simcore_service_webserver.projects.models import ProjectDict
 
 
@@ -24,6 +28,7 @@ class ProjectJobFixture:
     user_id: UserID
     project_uuid: ProjectID
     job_parent_resource_name: str
+    custom_metadata: MetadataDict = None
 
 
 @pytest.fixture
@@ -194,3 +199,125 @@ async def test_user_can_filter_marked_project(
     )
     assert total_count == 0
     assert len(result) == 0
+
+
+async def test_filter_projects_by_metadata(
+    client: TestClient,
+    logged_user: UserInfoDict,
+    user_project: ProjectDict,
+    osparc_product_name: ProductName,
+):
+    """Test that list_my_projects_marked_as_jobs can filter projects by custom metadata"""
+    assert client.app
+
+    user_id = logged_user["id"]
+    project_uuid = ProjectID(user_project["uuid"])
+    job_parent_resource_name = "test/resource/metadata"
+
+    # 1. Mark the project as a job
+    await set_project_as_job(
+        app=client.app,
+        product_name=osparc_product_name,
+        user_id=user_id,
+        project_uuid=project_uuid,
+        job_parent_resource_name=job_parent_resource_name,
+    )
+
+    # 2. Set custom metadata
+    custom_metadata: MetadataDict = {
+        "test_key": "test_value",
+        "category": "simulation",
+        "status": "completed",
+    }
+    await set_project_custom_metadata(
+        app=client.app,
+        user_id=user_id,
+        project_uuid=project_uuid,
+        value=custom_metadata,
+    )
+
+    # 3. Filter by exact metadata
+    filter_exact = [{"test_key": "test_value"}]
+    total_count, result = await list_my_projects_marked_as_jobs(
+        app=client.app,
+        product_name=osparc_product_name,
+        user_id=user_id,
+        filter_by_any_of_metadata=filter_exact,
+    )
+    assert total_count == 1
+    assert len(result) == 1
+    assert result[0].uuid == project_uuid
+
+    # 4. Filter by multiple metadata keys in one dict (AND condition)
+    filter_multiple_keys = [{"category": "simulation", "status": "completed"}]
+    total_count, result = await list_my_projects_marked_as_jobs(
+        app=client.app,
+        product_name=osparc_product_name,
+        user_id=user_id,
+        filter_by_any_of_metadata=filter_multiple_keys,
+    )
+    assert total_count == 1
+    assert len(result) == 1
+    assert result[0].uuid == project_uuid
+
+    # 5. Filter by alternative metadata (OR condition)
+    filter_alternative = [
+        {"status": "completed"},
+        {"status": "pending"},
+    ]
+    total_count, result = await list_my_projects_marked_as_jobs(
+        app=client.app,
+        product_name=osparc_product_name,
+        user_id=user_id,
+        filter_by_any_of_metadata=filter_alternative,
+    )
+    assert total_count == 1
+    assert len(result) == 1
+    assert result[0].uuid == project_uuid
+
+    # 6. Filter by non-matching metadata
+    filter_non_matching = [{"status": "failed"}]
+    total_count, result = await list_my_projects_marked_as_jobs(
+        app=client.app,
+        product_name=osparc_product_name,
+        user_id=user_id,
+        filter_by_any_of_metadata=filter_non_matching,
+    )
+    assert total_count == 0
+    assert len(result) == 0
+
+    # 7. Filter by wildcard pattern (requires SQL LIKE syntax)
+    # This assumes the implementation supports wildcards in metadata values
+    filter_wildcard = [{"test_key": "test_*"}]
+    total_count, result = await list_my_projects_marked_as_jobs(
+        app=client.app,
+        product_name=osparc_product_name,
+        user_id=user_id,
+        filter_by_any_of_metadata=filter_wildcard,
+    )
+    assert total_count == 1
+    assert len(result) == 1
+    assert result[0].uuid == project_uuid
+
+    # 8. Combine with parent resource name filter
+    total_count, result = await list_my_projects_marked_as_jobs(
+        app=client.app,
+        product_name=osparc_product_name,
+        user_id=user_id,
+        filter_by_job_parent_resource_name_prefix="test/resource",
+        filter_by_any_of_metadata=[{"category": "simulation"}],
+    )
+    assert total_count == 1
+    assert len(result) == 1
+    assert result[0].uuid == project_uuid
+
+    # 9. Conflicting filters should return no results
+    total_count, result = await list_my_projects_marked_as_jobs(
+        app=client.app,
+        product_name=osparc_product_name,
+        user_id=user_id,
+        filter_by_job_parent_resource_name_prefix="non-matching",
+        filter_by_any_of_metadata=[{"category": "simulation"}],
+    )
+    assert total_count == 0
+    assert len(result) == 0