diff --git a/packages/models-library/src/models_library/rpc/webserver/projects.py b/packages/models-library/src/models_library/rpc/webserver/projects.py index 195358a72132..d1bfff342134 100644 --- a/packages/models-library/src/models_library/rpc/webserver/projects.py +++ b/packages/models-library/src/models_library/rpc/webserver/projects.py @@ -2,12 +2,68 @@ from typing import Annotated, TypeAlias from uuid import uuid4 -from models_library.projects import NodesDict, ProjectID -from models_library.projects_nodes import Node -from models_library.rpc_pagination import PageRpc from pydantic import BaseModel, ConfigDict, Field from pydantic.config import JsonDict +from ...projects import NodesDict, ProjectID +from ...projects_nodes import Node +from ...rpc_pagination import PageRpc + + +class MetadataFilterItem(BaseModel): + name: str + pattern: str + + +class ListProjectsMarkedAsJobRpcFilters(BaseModel): + """Filters model for the list_projects_marked_as_jobs RPC. + + NOTE: Filters models are used to validate all possible filters in an API early on, + particularly to ensure compatibility and prevent conflicts between different filters. + """ + + job_parent_resource_name_prefix: str | None = None + + any_custom_metadata: Annotated[ + list[MetadataFilterItem] | None, + Field(description="Searchs for matches of any of the custom metadata fields"), + ] = None + + @staticmethod + def _update_json_schema_extra(schema: JsonDict) -> None: + schema.update( + { + "examples": [ + { + "job_parent_resource_name_prefix": "solvers/solver123", + "any_custom_metadata": [ + { + "name": "solver_type", + "pattern": "FEM", + }, + { + "name": "mesh_cells", + "pattern": "1*", + }, + ], + }, + { + "any_custom_metadata": [ + { + "name": "solver_type", + "pattern": "*CFD*", + } + ], + }, + {"job_parent_resource_name_prefix": "solvers/solver123"}, + ] + } + ) + + model_config = ConfigDict( + json_schema_extra=_update_json_schema_extra, + ) + class ProjectJobRpcGet(BaseModel): """ diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_rpc_server.py b/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_rpc_server.py index ca645218579a..17d8051d096e 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_rpc_server.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_rpc_server.py @@ -9,6 +9,7 @@ from models_library.projects import ProjectID from models_library.rest_pagination import PageOffsetInt from models_library.rpc.webserver.projects import ( + ListProjectsMarkedAsJobRpcFilters, PageRpcProjectJobRpcGet, ProjectJobRpcGet, ) @@ -56,24 +57,24 @@ async def list_projects_marked_as_jobs( # pagination offset: PageOffsetInt = 0, limit: PageLimitInt = DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, - # filters - job_parent_resource_name_prefix: str | None = None, + filters: ListProjectsMarkedAsJobRpcFilters | None = None, ) -> PageRpcProjectJobRpcGet: assert rpc_client assert product_name assert user_id - if job_parent_resource_name_prefix: - assert not job_parent_resource_name_prefix.startswith("/") - assert not job_parent_resource_name_prefix.endswith("%") - assert not job_parent_resource_name_prefix.startswith("%") + if filters and filters.job_parent_resource_name_prefix: + assert not filters.job_parent_resource_name_prefix.startswith("/") + assert not filters.job_parent_resource_name_prefix.endswith("%") + assert not filters.job_parent_resource_name_prefix.startswith("%") items = [ item for item in ProjectJobRpcGet.model_json_schema()["examples"] - if job_parent_resource_name_prefix is None + if filters is None + or filters.job_parent_resource_name_prefix is None or item.get("job_parent_resource_name").startswith( - job_parent_resource_name_prefix + filters.job_parent_resource_name_prefix ) ] diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/projects.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/projects.py index f2e261b7d6a4..15f40d66011e 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/projects.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/projects.py @@ -6,7 +6,10 @@ from models_library.projects import ProjectID from models_library.rabbitmq_basic_types import RPCMethodName from models_library.rest_pagination import PageOffsetInt -from models_library.rpc.webserver.projects import PageRpcProjectJobRpcGet +from models_library.rpc.webserver.projects import ( + ListProjectsMarkedAsJobRpcFilters, + PageRpcProjectJobRpcGet, +) from models_library.rpc_pagination import ( DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, PageLimitInt, @@ -51,8 +54,7 @@ async def list_projects_marked_as_jobs( # pagination offset: PageOffsetInt = 0, limit: PageLimitInt = DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, - # filters - job_parent_resource_name_prefix: str | None = None, + filters: ListProjectsMarkedAsJobRpcFilters | None = None, ) -> PageRpcProjectJobRpcGet: result = await rpc_client.request( WEBSERVER_RPC_NAMESPACE, @@ -61,7 +63,7 @@ async def list_projects_marked_as_jobs( user_id=user_id, offset=offset, limit=limit, - job_parent_resource_name_prefix=job_parent_resource_name_prefix, + filters=filters, ) assert TypeAdapter(PageRpcProjectJobRpcGet).validate_python(result) # nosec return cast(PageRpcProjectJobRpcGet, result) diff --git a/services/api-server/docs/api-server.drawio.svg b/services/api-server/docs/api-server.drawio.svg index 7491bb37fc37..3f6efeb32725 100644 --- a/services/api-server/docs/api-server.drawio.svg +++ b/services/api-server/docs/api-server.drawio.svg @@ -1,6 +1,6 @@ - + - + @@ -49,13 +49,13 @@ - + -
+
REPOSITORY @@ -225,6 +225,10 @@ + + + + @@ -246,6 +250,54 @@ + + + + + + + + + + + +
+
+
+ projects +
+
+
+ + + projects + + + + + + + + + + + +
+
+
+ postgres +
+ asyncpg +
+
+
+ + + postgres... + + + + @@ -471,7 +523,7 @@ - + @@ -500,8 +552,8 @@ - - + + @@ -527,13 +579,13 @@ - + -
+
simcore_service_catalog @@ -541,20 +593,20 @@
- + simcore_ser... - + -
+
simcore_service_webserver @@ -562,7 +614,7 @@
- + simcore_ser... @@ -744,15 +796,15 @@ - - + + - - + + - + @@ -805,8 +857,8 @@ - - + + @@ -832,13 +884,13 @@ - + -
+
Dependencies go inwards @@ -846,12 +898,16 @@
- + Dependencies go inwards + + + + diff --git a/services/api-server/src/simcore_service_api_server/_service_jobs.py b/services/api-server/src/simcore_service_api_server/_service_jobs.py index 35e268c17008..630c55b6c510 100644 --- a/services/api-server/src/simcore_service_api_server/_service_jobs.py +++ b/services/api-server/src/simcore_service_api_server/_service_jobs.py @@ -15,6 +15,7 @@ from models_library.users import UserID from pydantic import HttpUrl from servicelib.logging_utils import log_context +from simcore_service_api_server.models.basic_types import NameValueTuple from .models.schemas.jobs import Job, JobInputs from .models.schemas.programs import Program @@ -39,8 +40,9 @@ class JobService: async def list_jobs( self, + job_parent_resource_name: str, *, - filter_by_job_parent_resource_name_prefix: str, + filter_any_custom_metadata: list[NameValueTuple] | None = None, pagination_offset: PageOffsetInt = 0, pagination_limit: PageLimitInt = MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE - 1, ) -> tuple[list[Job], PageMetaInfoLimitOffset]: @@ -50,9 +52,10 @@ async def list_jobs( projects_page = await self._web_rpc_client.list_projects_marked_as_jobs( product_name=self.product_name, user_id=self.user_id, - offset=pagination_offset, - limit=pagination_limit, - job_parent_resource_name_prefix=filter_by_job_parent_resource_name_prefix, + pagination_offset=pagination_offset, + pagination_limit=pagination_limit, + filter_by_job_parent_resource_name_prefix=job_parent_resource_name, + filter_any_custom_metadata=filter_any_custom_metadata, ) # 2. Convert projects to jobs diff --git a/services/api-server/src/simcore_service_api_server/_service_solvers.py b/services/api-server/src/simcore_service_api_server/_service_solvers.py index 5b4fca17e2c3..afafe95ccf61 100644 --- a/services/api-server/src/simcore_service_api_server/_service_solvers.py +++ b/services/api-server/src/simcore_service_api_server/_service_solvers.py @@ -12,6 +12,7 @@ from models_library.services_enums import ServiceType from models_library.users import UserID from pydantic import NonNegativeInt, PositiveInt +from simcore_service_api_server.models.basic_types import NameValueTuple from ._service_jobs import JobService from ._service_utils import check_user_product_consistency @@ -92,6 +93,7 @@ async def list_jobs( *, filter_by_solver_key: SolverKeyId | None = None, filter_by_solver_version: VersionStr | None = None, + filter_any_custom_metadata: list[NameValueTuple] | None = None, pagination_offset: PageOffsetInt = 0, pagination_limit: PageLimitInt = DEFAULT_PAGINATION_LIMIT, ) -> tuple[list[Job], PageMetaInfoLimitOffset]: @@ -113,9 +115,10 @@ async def list_jobs( # 2. list jobs under job_parent_resource_name return await self.job_service.list_jobs( + job_parent_resource_name=job_parent_resource_name, + filter_any_custom_metadata=filter_any_custom_metadata, pagination_offset=pagination_offset, pagination_limit=pagination_limit, - filter_by_job_parent_resource_name_prefix=job_parent_resource_name, ) async def solver_release_history( diff --git a/services/api-server/src/simcore_service_api_server/_service_studies.py b/services/api-server/src/simcore_service_api_server/_service_studies.py index 61fb5894e53f..0fc31e7d4bb7 100644 --- a/services/api-server/src/simcore_service_api_server/_service_studies.py +++ b/services/api-server/src/simcore_service_api_server/_service_studies.py @@ -52,7 +52,7 @@ async def list_jobs( # 2. list jobs under job_parent_resource_name return await self.job_service.list_jobs( + job_parent_resource_name=job_parent_resource_name, pagination_offset=pagination_offset, pagination_limit=pagination_limit, - filter_by_job_parent_resource_name_prefix=job_parent_resource_name, ) diff --git a/services/api-server/src/simcore_service_api_server/api/dependencies/models_schemas_job_filters.py b/services/api-server/src/simcore_service_api_server/api/dependencies/models_schemas_job_filters.py new file mode 100644 index 000000000000..c4683f653d1c --- /dev/null +++ b/services/api-server/src/simcore_service_api_server/api/dependencies/models_schemas_job_filters.py @@ -0,0 +1,51 @@ +import textwrap +from typing import Annotated + +from fastapi import Query + +from ...models.schemas.jobs_filters import JobMetadataFilter, MetadataFilterItem + + +def get_job_metadata_filter( + any_: Annotated[ + list[str] | None, + Query( + alias="metadata.any", + description=textwrap.dedent( + """ + Filters jobs based on **any** of the matches on custom metadata fields. + + *Format*: `key:pattern` where pattern can contain glob wildcards + """ + ), + example=["key1:val*", "key2:exactval"], + ), + ] = None, +) -> JobMetadataFilter | None: + """ + Example input: + + /solvers/-/releases/-/jobs?metadata.any=key1:val*&metadata.any=key2:exactval + + This will be converted to: + JobMetadataFilter( + any=[ + MetadataFilterItem(name="key1", pattern="val*"), + MetadataFilterItem(name="key2", pattern="exactval"), + ] + ) + + This is used to filter jobs based on custom metadata fields. + + """ + if not any_: + return None + + items = [] + for item in any_: + try: + name, pattern = item.split(":", 1) + except ValueError: + continue # or raise HTTPException + items.append(MetadataFilterItem(name=name, pattern=pattern)) + return JobMetadataFilter(any=items) diff --git a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py index 250a8827cee9..bba01f54d7f8 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py @@ -25,7 +25,7 @@ from ...exceptions.custom_errors import InsufficientCreditsError, MissingWalletError from ...exceptions.service_errors_utils import DEFAULT_BACKEND_SERVICE_STATUS_CODES from ...models.api_resources import parse_resources_ids -from ...models.basic_types import LogStreamingResponse, VersionStr +from ...models.basic_types import LogStreamingResponse, NameValueTuple, VersionStr from ...models.domain.files import File as DomainFile from ...models.pagination import Page, PaginationParams from ...models.schemas.errors import ErrorGet @@ -38,6 +38,7 @@ JobMetadata, JobOutputs, ) +from ...models.schemas.jobs_filters import JobMetadataFilter from ...models.schemas.model_adapter import ( PricingUnitGetLegacy, WalletGetWithAvailableCreditsLegacy, @@ -55,6 +56,7 @@ from ..dependencies.application import get_reverse_url_mapper from ..dependencies.authentication import get_current_user_id from ..dependencies.database import get_db_asyncpg_engine +from ..dependencies.models_schemas_job_filters import get_job_metadata_filter from ..dependencies.rabbitmq import get_log_check_timeout, get_log_distributor from ..dependencies.services import get_api_client, get_solver_service from ..dependencies.webserver_http import AuthSession, get_webserver_session @@ -135,11 +137,22 @@ ) async def list_all_solvers_jobs( page_params: Annotated[PaginationParams, Depends()], + filter_job_metadata_params: Annotated[ + JobMetadataFilter | None, Depends(get_job_metadata_filter) + ], solver_service: Annotated[SolverService, Depends(get_solver_service)], url_for: Annotated[Callable, Depends(get_reverse_url_mapper)], ): jobs, meta = await solver_service.list_jobs( + filter_any_custom_metadata=( + [ + NameValueTuple(filter_metadata.name, filter_metadata.pattern) + for filter_metadata in filter_job_metadata_params.any + ] + if filter_job_metadata_params + else None + ), pagination_offset=page_params.offset, pagination_limit=page_params.limit, ) diff --git a/services/api-server/src/simcore_service_api_server/models/basic_types.py b/services/api-server/src/simcore_service_api_server/models/basic_types.py index 8e0c4c79af24..df9661bf0392 100644 --- a/services/api-server/src/simcore_service_api_server/models/basic_types.py +++ b/services/api-server/src/simcore_service_api_server/models/basic_types.py @@ -1,4 +1,4 @@ -from typing import Annotated, TypeAlias +from typing import Annotated, NamedTuple, TypeAlias from fastapi.responses import StreamingResponse from models_library.basic_regex import SIMPLE_VERSION_RE @@ -13,3 +13,8 @@ class LogStreamingResponse(StreamingResponse): media_type = "application/x-ndjson" + + +class NameValueTuple(NamedTuple): + name: str + value: str diff --git a/services/api-server/src/simcore_service_api_server/models/schemas/jobs_filters.py b/services/api-server/src/simcore_service_api_server/models/schemas/jobs_filters.py new file mode 100644 index 000000000000..961cae8d22f4 --- /dev/null +++ b/services/api-server/src/simcore_service_api_server/models/schemas/jobs_filters.py @@ -0,0 +1,58 @@ +from typing import Annotated + +from pydantic import BaseModel, ConfigDict, Field, StringConstraints +from pydantic.config import JsonDict + + +class MetadataFilterItem(BaseModel): + name: Annotated[ + str, + StringConstraints(min_length=1, max_length=255), + Field(description="Name fo the metadata field"), + ] + pattern: Annotated[ + str, + StringConstraints(min_length=1, max_length=255), + Field(description="Exact value or glob pattern"), + ] + + +class JobMetadataFilter(BaseModel): + any: Annotated[ + list[MetadataFilterItem], + Field(description="Matches any custom metadata field (OR logic)"), + ] + # NOTE: AND logic shall be implemented as `all: list[MetadataFilterItem] | None = None` + + @staticmethod + def _update_json_schema_extra(schema: JsonDict) -> None: + schema.update( + { + "examples": [ + { + "any": [ + { + "name": "solver_type", + "pattern": "FEM", + }, + { + "name": "mesh_cells", + "pattern": "1*", + }, + ] + }, + { + "any": [ + { + "name": "solver_type", + "pattern": "*CFD*", + } + ] + }, + ] + } + ) + + model_config = ConfigDict( + json_schema_extra=_update_json_schema_extra, + ) diff --git a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py index 582fccb9fa1d..a0bd4cf3e8fc 100644 --- a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py +++ b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py @@ -33,6 +33,10 @@ PageMetaInfoLimitOffset, PageOffsetInt, ) +from models_library.rpc.webserver.projects import ( + ListProjectsMarkedAsJobRpcFilters, + MetadataFilterItem, +) from models_library.services_types import ServiceRunID from models_library.users import UserID from models_library.wallets import WalletID @@ -66,6 +70,7 @@ from servicelib.rabbitmq.rpc_interfaces.webserver.licenses.licensed_items import ( release_licensed_item_for_wallet as _release_licensed_item_for_wallet, ) +from simcore_service_api_server.models.basic_types import NameValueTuple from ..exceptions.backend_errors import ( CanNotCheckoutServiceIsNotRunningError, @@ -243,17 +248,30 @@ async def list_projects_marked_as_jobs( *, product_name: ProductName, user_id: UserID, - offset: int = 0, - limit: int = 50, - job_parent_resource_name_prefix: str | None = None, + pagination_offset: int = 0, + pagination_limit: int = 50, + filter_by_job_parent_resource_name_prefix: str | None, + filter_any_custom_metadata: list[NameValueTuple] | None, ): + filters = ListProjectsMarkedAsJobRpcFilters( + job_parent_resource_name_prefix=filter_by_job_parent_resource_name_prefix, + any_custom_metadata=( + [ + MetadataFilterItem(name=name, pattern=pattern) + for name, pattern in filter_any_custom_metadata + ] + if filter_any_custom_metadata + else None + ), + ) + return await projects_rpc.list_projects_marked_as_jobs( rpc_client=self._client, product_name=product_name, user_id=user_id, - offset=offset, - limit=limit, - job_parent_resource_name_prefix=job_parent_resource_name_prefix, + offset=pagination_offset, + limit=pagination_limit, + filters=filters, ) async def register_function(self, *, function: Function) -> RegisteredFunction: diff --git a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_read.py b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_read.py index b7dca6511a62..57b69e001d36 100644 --- a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_read.py +++ b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_read.py @@ -7,6 +7,7 @@ import httpx import pytest +from models_library.users import UserID from pydantic import TypeAdapter from pytest_mock import MockType from pytest_simcore.helpers.httpx_calls_capture_models import HttpApiCallCaptureModel @@ -124,3 +125,62 @@ async def test_list_all_solvers_jobs( assert job.outputs_url is not None assert mocked_backend.webserver_rpc["list_projects_marked_as_jobs"].called + + +async def test_list_all_solvers_jobs_with_metadata_filter( + auth: httpx.BasicAuth, + client: httpx.AsyncClient, + mocked_backend: MockBackendRouters, + user_id: UserID, +): + """Tests the endpoint that lists all jobs across all solvers with metadata filtering.""" + + # Test with metadata filters + metadata_filters = ["key1:val*", "key2:exactval"] + + # Construct query parameters with metadata.any filters + params = { + "limit": 10, + "offset": 0, + "metadata.any": metadata_filters, + } + + # Call the endpoint with metadata filters + resp = await client.get( + f"/{API_VTAG}/solvers/-/releases/-/jobs", + auth=auth, + params=params, + ) + + # Verify the response + assert resp.status_code == status.HTTP_200_OK + + # Parse and validate the response + jobs_page = TypeAdapter(Page[Job]).validate_python(resp.json()) + + # Basic assertions on the response structure + assert isinstance(jobs_page.items, list) + assert jobs_page.limit == 10 + assert jobs_page.offset == 0 + + # Check that the backend was called with the correct filter parameters + assert mocked_backend.webserver_rpc["list_projects_marked_as_jobs"].called + + # Get the call args to verify filter parameters were passed correctly + call_args = mocked_backend.webserver_rpc["list_projects_marked_as_jobs"].call_args + + # The filter_any_custom_metadata parameter should contain our filters + # The exact structure will depend on how your mocked function is called + assert call_args is not None + + assert call_args.kwargs["product_name"] == "osparc" + assert call_args.kwargs["user_id"] == user_id + assert call_args.kwargs["offset"] == 0 + assert call_args.kwargs["limit"] == 10 + assert call_args.kwargs["filters"] + + # Verify the metadata filters were correctly transformed and passed + assert call_args.kwargs["filters"].any_custom_metadata[0].name == "key1" + assert call_args.kwargs["filters"].any_custom_metadata[0].pattern == "val*" + assert call_args.kwargs["filters"].any_custom_metadata[1].name == "key2" + assert call_args.kwargs["filters"].any_custom_metadata[1].pattern == "exactval" diff --git a/services/api-server/tests/unit/service/test_service_jobs.py b/services/api-server/tests/unit/service/test_service_jobs.py index 01947ba1cd8e..d6829339507f 100644 --- a/services/api-server/tests/unit/service/test_service_jobs.py +++ b/services/api-server/tests/unit/service/test_service_jobs.py @@ -16,7 +16,7 @@ async def test_list_jobs_by_resource_prefix( ): # Test with default pagination parameters jobs, page_meta = await job_service.list_jobs( - filter_by_job_parent_resource_name_prefix="solvers/some-solver" + job_parent_resource_name="solvers/some-solver" ) assert isinstance(jobs, list) diff --git a/services/api-server/tests/unit/service/test_service_studies.py b/services/api-server/tests/unit/service/test_service_studies.py index dd7d2c851eb1..fa9b9921866e 100644 --- a/services/api-server/tests/unit/service/test_service_studies.py +++ b/services/api-server/tests/unit/service/test_service_studies.py @@ -28,7 +28,9 @@ async def test_list_jobs_no_study_id( # Verify proper prefix was used assert ( - mocked_rpc_client.request.call_args.kwargs["job_parent_resource_name_prefix"] + mocked_rpc_client.request.call_args.kwargs[ + "filters" + ].job_parent_resource_name_prefix == "study" ) @@ -49,7 +51,9 @@ async def test_list_jobs_with_study_id( # Verify proper prefix was used with study ID assert ( - mocked_rpc_client.request.call_args.kwargs["job_parent_resource_name_prefix"] + mocked_rpc_client.request.call_args.kwargs[ + "filters" + ].job_parent_resource_name_prefix == f"study/{study_id}" ) diff --git a/services/api-server/tests/unit/test_api_dependencies.py b/services/api-server/tests/unit/test_api_dependencies.py new file mode 100644 index 000000000000..2fb1f506008d --- /dev/null +++ b/services/api-server/tests/unit/test_api_dependencies.py @@ -0,0 +1,146 @@ +from typing import Annotated + +import pytest +from fastapi import Depends, FastAPI, status +from fastapi.testclient import TestClient +from pydantic import ValidationError +from simcore_service_api_server.api.dependencies.models_schemas_job_filters import ( + get_job_metadata_filter, +) +from simcore_service_api_server.models.schemas.jobs_filters import ( + JobMetadataFilter, + MetadataFilterItem, +) + + +def test_get_metadata_filter(): + # Test with None input + assert get_job_metadata_filter(None) is None + + # Test with empty list + assert get_job_metadata_filter([]) is None + + # Test with valid input (matching the example in the docstring) + input_data = ["key1:val*", "key2:exactval"] + result = get_job_metadata_filter(input_data) + + expected = JobMetadataFilter( + any=[ + MetadataFilterItem(name="key1", pattern="val*"), + MetadataFilterItem(name="key2", pattern="exactval"), + ] + ) + + assert result is not None + assert len(result.any) == 2 + assert result.any[0].name == "key1" + assert result.any[0].pattern == "val*" + assert result.any[1].name == "key2" + assert result.any[1].pattern == "exactval" + assert result == expected + + # Test with invalid input (missing colon) + input_data = ["key1val", "key2:exactval"] + result = get_job_metadata_filter(input_data) + + assert result is not None + assert len(result.any) == 1 + assert result.any[0].name == "key2" + assert result.any[0].pattern == "exactval" + + # Test with empty pattern not allowed + input_data = ["key1:", "key2:exactval"] + with pytest.raises(ValidationError) as exc_info: + get_job_metadata_filter(input_data) + + assert exc_info.value.errors()[0]["type"] == "string_too_short" + + +def test_metadata_filter_in_api_route(): + # Create a test FastAPI app + app = FastAPI() + + # Define a route that uses the get_metadata_filter dependency + @app.get("/test-filter") + def filter_endpoint( + metadata_filter: Annotated[ + JobMetadataFilter | None, Depends(get_job_metadata_filter) + ] = None, + ): + if not metadata_filter: + return {"filters": None} + + # Convert to dict for easier comparison in test + return { + "filters": { + "any": [ + {"name": item.name, "pattern": item.pattern} + for item in metadata_filter.any + ] + } + } + + # Create a test client + client = TestClient(app) + + # Test with no filter + response = client.get("/test-filter") + assert response.status_code == status.HTTP_200_OK + assert response.json() == {"filters": None} + + # Test with single filter + response = client.get("/test-filter?metadata.any=key1:val*") + assert response.status_code == status.HTTP_200_OK + assert response.json() == { + "filters": {"any": [{"name": "key1", "pattern": "val*"}]} + } + + # Test with multiple filters + response = client.get( + "/test-filter?metadata.any=key1:val*&metadata.any=key2:exactval" + ) + assert response.status_code == status.HTTP_200_OK + assert response.json() == { + "filters": { + "any": [ + {"name": "key1", "pattern": "val*"}, + {"name": "key2", "pattern": "exactval"}, + ] + } + } + + # Test with invalid filter (should skip the invalid one) + response = client.get( + "/test-filter?metadata.any=invalid&metadata.any=key2:exactval" + ) + assert response.status_code == status.HTTP_200_OK + assert response.json() == { + "filters": {"any": [{"name": "key2", "pattern": "exactval"}]} + } + + # Test with URL-encoded characters + # Use special characters that need encoding: space, &, =, +, /, ? + encoded_query = "/test-filter?metadata.any=special%20key:value%20with%20spaces&metadata.any=symbols:a%2Bb%3Dc%26d%3F%2F" + response = client.get(encoded_query) + assert response.status_code == status.HTTP_200_OK + assert response.json() == { + "filters": { + "any": [ + {"name": "special key", "pattern": "value with spaces"}, + {"name": "symbols", "pattern": "a+b=c&d?/"}, + ] + } + } + + # Test with Unicode characters + unicode_query = "/test-filter?metadata.any=emoji:%F0%9F%98%8A&metadata.any=international:caf%C3%A9" + response = client.get(unicode_query) + assert response.status_code == status.HTTP_200_OK + assert response.json() == { + "filters": { + "any": [ + {"name": "emoji", "pattern": "😊"}, + {"name": "international", "pattern": "café"}, + ] + } + } diff --git a/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rpc.py b/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rpc.py index 6ee4861b6902..e511d4dd4980 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rpc.py +++ b/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rpc.py @@ -4,6 +4,7 @@ from models_library.projects import ProjectID from models_library.rest_pagination import PageLimitInt, PageOffsetInt from models_library.rpc.webserver.projects import ( + ListProjectsMarkedAsJobRpcFilters, PageRpcProjectJobRpcGet, ProjectJobRpcGet, ) @@ -65,17 +66,26 @@ async def list_projects_marked_as_jobs( # pagination offset: PageOffsetInt, limit: PageLimitInt, - # filters - job_parent_resource_name_prefix: str | None, + filters: ListProjectsMarkedAsJobRpcFilters | None = None, ) -> PageRpcProjectJobRpcGet: total, projects = await _jobs_service.list_my_projects_marked_as_jobs( app, product_name=product_name, user_id=user_id, - offset=offset, - limit=limit, - job_parent_resource_name_prefix=job_parent_resource_name_prefix, + pagination_offset=offset, + pagination_limit=limit, + filter_by_job_parent_resource_name_prefix=( + filters.job_parent_resource_name_prefix if filters else None + ), + filter_any_custom_metadata=( + [ + (custom_metadata.name, custom_metadata.pattern) + for custom_metadata in filters.any_custom_metadata + ] + if filters and filters.any_custom_metadata + else None + ), ) job_projects = [ diff --git a/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py b/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py index b6fd3242c019..3b060be9a23b 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py +++ b/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py @@ -8,6 +8,7 @@ from simcore_postgres_database.models.groups import user_to_groups from simcore_postgres_database.models.project_to_groups import project_to_groups from simcore_postgres_database.models.projects import projects +from simcore_postgres_database.models.projects_metadata import projects_metadata from simcore_postgres_database.models.projects_to_jobs import projects_to_jobs from simcore_postgres_database.models.projects_to_products import projects_to_products from simcore_postgres_database.utils_repos import ( @@ -30,6 +31,34 @@ ) +def _apply_job_parent_resource_name_filter( + query: sa.sql.Select, prefix: str +) -> sa.sql.Select: + return query.where(projects_to_jobs.c.job_parent_resource_name.like(f"{prefix}%")) + + +def _apply_custom_metadata_filter( + query: sa.sql.Select, any_metadata_fields: list[tuple[str, str]] +) -> sa.sql.Select: + """Apply metadata filters to query. + + For PostgreSQL JSONB fields, we need to extract the text value using ->> operator + before applying string comparison operators like ILIKE. + """ + assert any_metadata_fields # nosec + + metadata_fields_ilike = [] + for key, pattern in any_metadata_fields: + # Use ->> operator to extract the text value from JSONB + # Then apply ILIKE for case-insensitive pattern matching + sql_pattern = pattern.replace("*", "%") # Convert glob-like pattern to SQL LIKE + metadata_fields_ilike.append( + projects_metadata.c.custom[key].astext.ilike(sql_pattern) + ) + + return query.where(sa.or_(*metadata_fields_ilike)) + + class ProjectJobsRepository(BaseRepository): async def set_project_as_job( @@ -60,9 +89,10 @@ async def list_projects_marked_as_jobs( *, product_name: ProductName, user_id: UserID, - offset: int = 0, - limit: int = 10, - job_parent_resource_name_prefix: str | None = None, + pagination_offset: int, + pagination_limit: int, + filter_by_job_parent_resource_name_prefix: str | None = None, + filter_any_custom_metadata: list[tuple[str, str]] | None = None, ) -> tuple[int, list[ProjectJobDBGet]]: """Lists projects marked as jobs for a specific user and product @@ -96,10 +126,20 @@ async def list_projects_marked_as_jobs( projects_to_products, projects_to_jobs.c.project_uuid == projects_to_products.c.project_uuid, - ).join( + ) + .join( project_to_groups, projects_to_jobs.c.project_uuid == project_to_groups.c.project_uuid, ) + .join( + # NOTE: avoids `SAWarning: SELECT statement has a cartesian product ...` + projects, + projects_to_jobs.c.project_uuid == projects.c.uuid, + ) + .outerjoin( + projects_metadata, + projects_to_jobs.c.project_uuid == projects_metadata.c.project_uuid, + ) ) .where( projects_to_products.c.product_name == product_name, @@ -112,21 +152,24 @@ async def list_projects_marked_as_jobs( ) ) - # Apply job_parent_resource_name_filter if provided - if job_parent_resource_name_prefix: - access_query = access_query.where( - projects_to_jobs.c.job_parent_resource_name.like( - f"{job_parent_resource_name_prefix}%" - ) + # Step 3: Apply filters + if filter_by_job_parent_resource_name_prefix: + access_query = _apply_job_parent_resource_name_filter( + access_query, filter_by_job_parent_resource_name_prefix + ) + + if filter_any_custom_metadata: + access_query = _apply_custom_metadata_filter( + access_query, filter_any_custom_metadata ) - # Convert access_query to a subquery + # Step 4. Convert access_query to a subquery base_query = access_query.subquery() - # Step 3: Query to get the total count + # Step 5: Query to get the total count total_query = sa.select(sa.func.count()).select_from(base_query) - # Step 4: Query to get the paginated list with full selection + # Step 6: Query to get the paginated list with full selection list_query = ( sa.select( *_PROJECT_DB_COLS, @@ -143,8 +186,8 @@ async def list_projects_marked_as_jobs( projects.c.creation_date.desc(), # latests first projects.c.id.desc(), ) - .limit(limit) - .offset(offset) + .limit(pagination_limit) + .offset(pagination_offset) ) # Step 5: Execute queries diff --git a/services/web/server/src/simcore_service_webserver/projects/_jobs_service.py b/services/web/server/src/simcore_service_webserver/projects/_jobs_service.py index 7b4289644c28..6fa7a88fa703 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_jobs_service.py +++ b/services/web/server/src/simcore_service_webserver/projects/_jobs_service.py @@ -54,19 +54,27 @@ async def list_my_projects_marked_as_jobs( *, product_name: ProductName, user_id: UserID, - offset: int = 0, - limit: int = 10, - job_parent_resource_name_prefix: str | None = None, + pagination_offset: int = 0, + pagination_limit: int = 10, + filter_by_job_parent_resource_name_prefix: str | None = None, + filter_any_custom_metadata: list[tuple[str, str]] | None = None, ) -> tuple[int, list[ProjectJobDBGet]]: """ Lists paginated projects marked as jobs for the given user and product. - Optionally filters by job_parent_resource_name using SQL-like wildcard patterns. + + Keyword Arguments: + filter_by_job_parent_resource_name_prefix -- Optionally filters by job_parent_resource_name using SQL-like wildcard patterns. (default: {None}) + filter_any_custom_metadata -- is a list of dictionaries with key-pattern pairs for custom metadata fields (OR logic). (default: {None}) + + Returns: + A tuple containing the total number of projects and a list of ProjectJobDBGet objects for this page. """ repo = ProjectJobsRepository.create_from_app(app) return await repo.list_projects_marked_as_jobs( user_id=user_id, product_name=product_name, - offset=offset, - limit=limit, - job_parent_resource_name_prefix=job_parent_resource_name_prefix, + pagination_offset=pagination_offset, + pagination_limit=pagination_limit, + filter_by_job_parent_resource_name_prefix=filter_by_job_parent_resource_name_prefix, + filter_any_custom_metadata=filter_any_custom_metadata, ) diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects__jobs_service.py b/services/web/server/tests/unit/with_dbs/02/test_projects__jobs_service.py index ebe796ad9f0e..df62390bf37e 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects__jobs_service.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects__jobs_service.py @@ -8,6 +8,7 @@ import pytest from aiohttp.test_utils import TestClient from common_library.users_enums import UserRole +from models_library.api_schemas_webserver.projects_metadata import MetadataDict from models_library.products import ProductName from models_library.projects import ProjectID from models_library.users import UserID @@ -16,6 +17,9 @@ list_my_projects_marked_as_jobs, set_project_as_job, ) +from simcore_service_webserver.projects._metadata_service import ( + set_project_custom_metadata, +) from simcore_service_webserver.projects.models import ProjectDict @@ -24,6 +28,7 @@ class ProjectJobFixture: user_id: UserID project_uuid: ProjectID job_parent_resource_name: str + custom_metadata: MetadataDict | None = None @pytest.fixture @@ -158,7 +163,7 @@ async def test_user_can_filter_marked_project( app=client.app, product_name=osparc_product_name, user_id=project_job_fixture.user_id, - job_parent_resource_name_prefix=project_job_fixture.job_parent_resource_name, + filter_by_job_parent_resource_name_prefix=project_job_fixture.job_parent_resource_name, ) assert total_count == 1 assert len(result) == 1 @@ -174,7 +179,7 @@ async def test_user_can_filter_marked_project( app=client.app, product_name=osparc_product_name, user_id=project_job_fixture.user_id, - job_parent_resource_name_prefix="test/%", + filter_by_job_parent_resource_name_prefix="test/%", ) assert total_count == 1 assert len(result) == 1 @@ -190,7 +195,132 @@ async def test_user_can_filter_marked_project( app=client.app, product_name=osparc_product_name, user_id=project_job_fixture.user_id, - job_parent_resource_name_prefix="other/%", + filter_by_job_parent_resource_name_prefix="other/%", + ) + assert total_count == 0 + assert len(result) == 0 + + +async def test_filter_projects_by_metadata( + client: TestClient, + logged_user: UserInfoDict, + user_project: ProjectDict, + osparc_product_name: ProductName, +): + """Test that list_my_projects_marked_as_jobs can filter projects by custom metadata""" + assert client.app + + user_id = logged_user["id"] + project_uuid = ProjectID(user_project["uuid"]) + job_parent_resource_name = "test/resource/metadata" + + # 1. Mark the project as a job + await set_project_as_job( + app=client.app, + product_name=osparc_product_name, + user_id=user_id, + project_uuid=project_uuid, + job_parent_resource_name=job_parent_resource_name, + ) + + # 2. Set custom metadata + custom_metadata: MetadataDict = { + "test_key": "test_value", + "category": "simulation", + "status": "completed", + } + await set_project_custom_metadata( + app=client.app, + user_id=user_id, + project_uuid=project_uuid, + value=custom_metadata, + ) + + # 3. Filter by exact metadata + filter_exact = [("test_key", "test_value")] + total_count, result = await list_my_projects_marked_as_jobs( + app=client.app, + product_name=osparc_product_name, + user_id=user_id, + filter_any_custom_metadata=filter_exact, + ) + assert total_count == 1 + assert len(result) == 1 + assert result[0].uuid == project_uuid + + # 4. Filter by multiple metadata keys in one dict (AND condition) + filter_multiple_keys = [ + ("category", "simulation"), + ("status", "completed"), + ] + total_count, result = await list_my_projects_marked_as_jobs( + app=client.app, + product_name=osparc_product_name, + user_id=user_id, + filter_any_custom_metadata=filter_multiple_keys, + ) + assert total_count == 1 + assert len(result) == 1 + assert result[0].uuid == project_uuid + + # 5. Filter by alternative metadata (OR condition) + filter_alternative = [ + ("status", "completed"), + ("status", "pending"), + ] + total_count, result = await list_my_projects_marked_as_jobs( + app=client.app, + product_name=osparc_product_name, + user_id=user_id, + filter_any_custom_metadata=filter_alternative, + ) + assert total_count == 1 + assert len(result) == 1 + assert result[0].uuid == project_uuid + + # 6. Filter by non-matching metadata + filter_non_matching = [("status", "failed")] + total_count, result = await list_my_projects_marked_as_jobs( + app=client.app, + product_name=osparc_product_name, + user_id=user_id, + filter_any_custom_metadata=filter_non_matching, + ) + assert total_count == 0 + assert len(result) == 0 + + # 7. Filter by wildcard pattern (requires SQL LIKE syntax) + # This assumes the implementation supports wildcards in metadata values + filter_wildcard = [("test_key", "test_*")] + total_count, result = await list_my_projects_marked_as_jobs( + app=client.app, + product_name=osparc_product_name, + user_id=user_id, + filter_any_custom_metadata=filter_wildcard, + ) + assert total_count == 1 + assert len(result) == 1 + assert result[0].uuid == project_uuid + + # 8. Combine with parent resource name filter + total_count, result = await list_my_projects_marked_as_jobs( + app=client.app, + product_name=osparc_product_name, + user_id=user_id, + filter_by_job_parent_resource_name_prefix="test/resource", + filter_any_custom_metadata=[("category", "simulation")], + ) + assert total_count == 1 + assert len(result) == 1 + assert result[0].uuid == project_uuid + + # 9. Conflicting filters should return no results + total_count, result = await list_my_projects_marked_as_jobs( + app=client.app, + product_name=osparc_product_name, + user_id=user_id, + filter_by_job_parent_resource_name_prefix="non-matching", + filter_any_custom_metadata=[("category", "simulation")], ) assert total_count == 0 assert len(result) == 0 diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_rpc.py b/services/web/server/tests/unit/with_dbs/02/test_projects_rpc.py index 28147c93d3eb..ac38c6061d4c 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_rpc.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_rpc.py @@ -13,6 +13,8 @@ from models_library.products import ProductName from models_library.projects import ProjectID from models_library.rpc.webserver.projects import ( + ListProjectsMarkedAsJobRpcFilters, + MetadataFilterItem, PageRpcProjectJobRpcGet, ProjectJobRpcGet, ) @@ -114,7 +116,9 @@ async def test_rpc_client_list_my_projects_marked_as_jobs( rpc_client=rpc_client, product_name=product_name, user_id=user_id, - job_parent_resource_name_prefix="solvers/solver123", + filters=ListProjectsMarkedAsJobRpcFilters( + job_parent_resource_name_prefix="solvers/solver123" + ), ) assert page.meta.total == 1 @@ -191,3 +195,115 @@ async def test_errors_on_rpc_client_mark_project_as_job( assert exc_info.value.error_count() == 1 assert exc_info.value.errors()[0]["loc"] == ("job_parent_resource_name",) assert exc_info.value.errors()[0]["type"] == "value_error" + + +async def test_rpc_client_list_projects_marked_as_jobs_with_metadata_filter( + rpc_client: RabbitMQRPCClient, + product_name: ProductName, + logged_user: UserInfoDict, + user_project: ProjectDict, + client: TestClient, +): + from simcore_service_webserver.projects import _metadata_service + + project_uuid = ProjectID(user_project["uuid"]) + user_id = logged_user["id"] + + # Mark the project as a job + await projects_rpc.mark_project_as_job( + rpc_client=rpc_client, + product_name=product_name, + user_id=user_id, + project_uuid=project_uuid, + job_parent_resource_name="solvers/solver123/version/1.2.3", + ) + + # Set custom metadata on the project + custom_metadata = { + "solver_type": "FEM", + "mesh_cells": "10000", + "domain": "biomedical", + } + + await _metadata_service.set_project_custom_metadata( + app=client.app, + user_id=user_id, + project_uuid=project_uuid, + value=custom_metadata, + ) + + # Test with exact match on metadata field + page: PageRpcProjectJobRpcGet = await projects_rpc.list_projects_marked_as_jobs( + rpc_client=rpc_client, + product_name=product_name, + user_id=user_id, + filters=ListProjectsMarkedAsJobRpcFilters( + job_parent_resource_name_prefix="solvers/solver123", + any_custom_metadata=[MetadataFilterItem(name="solver_type", pattern="FEM")], + ), + ) + + assert page.meta.total == 1 + assert len(page.data) == 1 + assert page.data[0].uuid == project_uuid + + # Test with pattern match on metadata field + page = await projects_rpc.list_projects_marked_as_jobs( + rpc_client=rpc_client, + product_name=product_name, + user_id=user_id, + filters=ListProjectsMarkedAsJobRpcFilters( + any_custom_metadata=[MetadataFilterItem(name="mesh_cells", pattern="1*")], + ), + ) + + assert page.meta.total == 1 + assert len(page.data) == 1 + assert page.data[0].uuid == project_uuid + + # Test with multiple metadata fields (any match should return the project) + page = await projects_rpc.list_projects_marked_as_jobs( + rpc_client=rpc_client, + product_name=product_name, + user_id=user_id, + filters=ListProjectsMarkedAsJobRpcFilters( + any_custom_metadata=[ + MetadataFilterItem(name="solver_type", pattern="FEM"), + MetadataFilterItem(name="non_existent", pattern="value"), + ], + ), + ) + + assert page.meta.total == 1 + assert len(page.data) == 1 + + # Test with no matches + page = await projects_rpc.list_projects_marked_as_jobs( + rpc_client=rpc_client, + product_name=product_name, + user_id=user_id, + filters=ListProjectsMarkedAsJobRpcFilters( + any_custom_metadata=[ + MetadataFilterItem(name="solver_type", pattern="CFD"), # No match + ], + ), + ) + + assert page.meta.total == 0 + assert len(page.data) == 0 + + # Test with combination of resource prefix and metadata + page = await projects_rpc.list_projects_marked_as_jobs( + rpc_client=rpc_client, + product_name=product_name, + user_id=user_id, + filters=ListProjectsMarkedAsJobRpcFilters( + job_parent_resource_name_prefix="wrong/prefix", + any_custom_metadata=[ + MetadataFilterItem(name="solver_type", pattern="FEM"), + ], + ), + ) + + assert page.meta.total == 0 + assert len(page.data) == 0