adds filters to the public rest API

Author: pcrespov

packages/models-library/src/models_library/rpc/webserver/projects.py

@@ -2,7 +2,7 @@
 from typing import Annotated, TypeAlias
 from uuid import uuid4
 
-from pydantic import BaseModel, ConfigDict, Field, StringConstraints
+from pydantic import BaseModel, ConfigDict, Field
 from pydantic.config import JsonDict
 
 from ...projects import NodesDict, ProjectID
@@ -11,16 +11,8 @@
 
 
 class MetadataFilterItem(BaseModel):
-    name: Annotated[
-        str,
-        StringConstraints(min_length=1, max_length=255),
-        Field(description="Name fo the custom metadata field"),
-    ]
-    pattern: Annotated[
-        str,
-        StringConstraints(min_length=1, max_length=255),
-        Field(description="Exact value or glob pattern"),
-    ]
+    name: str
+    pattern: str
 
 
 class ListProjectsMarkedAsJobRpcFilter(BaseModel):

services/api-server/src/simcore_service_api_server/_service_solvers.py

@@ -92,6 +92,7 @@ async def list_jobs(
         *,
         filter_by_solver_key: SolverKeyId | None = None,
         filter_by_solver_version: VersionStr | None = None,
+        filter_by_job_custom_metadata: list[dict[str, str]] | None = None,
         pagination_offset: PageOffsetInt = 0,
         pagination_limit: PageLimitInt = DEFAULT_PAGINATION_LIMIT,
     ) -> tuple[list[Job], PageMetaInfoLimitOffset]:
@@ -114,6 +115,7 @@ async def list_jobs(
         # 2. list jobs under job_parent_resource_name
         return await self.job_service.list_jobs(
             job_parent_resource_name=job_parent_resource_name,
+            filter_by_job_custom_metadata=filter_by_job_custom_metadata,
             pagination_offset=pagination_offset,
             pagination_limit=pagination_limit,
         )

services/api-server/src/simcore_service_api_server/api/dependencies/models_schemas_job_filters.py

@@ -0,0 +1,51 @@
+import textwrap
+from typing import Annotated
+
+from fastapi import Query
+
+from ...models.schemas.jobs_filters import JobMetadataFilter, MetadataFilterItem
+
+
+def get_job_metadata_filter(
+    any_: Annotated[
+        list[str] | None,
+        Query(
+            alias="metadata.any",
+            description=textwrap.dedent(
+                """
+            Filters jobs based on **any** of the matches on custom metadata fields.
+
+            *Format*: `key:pattern` where pattern can contain glob wildcards
+            """
+            ),
+            example=["key1:val*", "key2:exactval"],
+        ),
+    ] = None,
+) -> JobMetadataFilter | None:
+    """
+    Example input:
+
+        /solvers/-/releases/-/jobs?metadata.any=key1:val*&metadata.any=key2:exactval
+
+    This will be converted to:
+        JobMetadataFilter(
+            any=[
+                MetadataFilterItem(name="key1", pattern="val*"),
+                MetadataFilterItem(name="key2", pattern="exactval"),
+            ]
+        )
+
+    This is used to filter jobs based on custom metadata fields.
+
+    """
+    if not any_:
+        return None
+
+    items = []
+    for item in any_:
+        try:
+            name, pattern = item.split(":", 1)
+        except ValueError:
+            continue  # or raise HTTPException
+        items.append(MetadataFilterItem(name=name, pattern=pattern))
+    return JobMetadataFilter(any=items)

services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py

@@ -18,6 +18,7 @@
 from pydantic import HttpUrl, NonNegativeInt
 from pydantic.types import PositiveInt
 from servicelib.logging_utils import log_context
+from simcore_service_api_server.models.schemas.jobs_filters import JobMetadataFilter
 from sqlalchemy.ext.asyncio import AsyncEngine
 from starlette.background import BackgroundTask
 
@@ -38,6 +39,7 @@
     JobMetadata,
     JobOutputs,
 )
+from ...models.schemas.jobs_filters import JobMetadataFilter
 from ...models.schemas.model_adapter import (
     PricingUnitGetLegacy,
     WalletGetWithAvailableCreditsLegacy,
@@ -55,6 +57,7 @@
 from ..dependencies.application import get_reverse_url_mapper
 from ..dependencies.authentication import get_current_user_id
 from ..dependencies.database import get_db_asyncpg_engine
+from ..dependencies.models_schemas_job_filters import get_job_metadata_filter
 from ..dependencies.rabbitmq import get_log_check_timeout, get_log_distributor
 from ..dependencies.services import get_api_client, get_solver_service
 from ..dependencies.webserver_http import AuthSession, get_webserver_session
@@ -131,15 +134,26 @@
             FMSG_CHANGELOG_NEW_IN_VERSION.format("0.8"),
         ],
     ),
-    include_in_schema=False,  # TO BE RELEASED in 0.8
+    include_in_schema=True,  # TO BE RELEASED in 0.8
 )
 async def list_all_solvers_jobs(
     page_params: Annotated[PaginationParams, Depends()],
+    filter_job_metadata_params: Annotated[
+        JobMetadataFilter | None, Depends(get_job_metadata_filter)
+    ],
     solver_service: Annotated[SolverService, Depends(get_solver_service)],
     url_for: Annotated[Callable, Depends(get_reverse_url_mapper)],
 ):
 
     jobs, meta = await solver_service.list_jobs(
+        filter_by_job_custom_metadata=(
+            [
+                {filter_metadata.name: filter_metadata.pattern}
+                for filter_metadata in filter_job_metadata_params.any
+            ]
+            if filter_job_metadata_params
+            else None
+        ),
         pagination_offset=page_params.offset,
         pagination_limit=page_params.limit,
     )

services/api-server/src/simcore_service_api_server/models/schemas/jobs_filters.py

@@ -0,0 +1,57 @@
+from typing import Annotated
+
+from pydantic import BaseModel, ConfigDict, Field, StringConstraints
+from pydantic.config import JsonDict
+
+
+class MetadataFilterItem(BaseModel):
+    name: Annotated[
+        str,
+        StringConstraints(min_length=1, max_length=255),
+        Field(description="Name fo the metadata field"),
+    ]
+    pattern: Annotated[
+        str,
+        StringConstraints(min_length=1, max_length=255),
+        Field(description="Exact value or glob pattern"),
+    ]
+
+
+class JobMetadataFilter(BaseModel):
+    any: Annotated[
+        list[MetadataFilterItem],
+        Field(description="Matches any custom metadata field (OR logic)"),
+    ]
+
+    @staticmethod
+    def _update_json_schema_extra(schema: JsonDict) -> None:
+        schema.update(
+            {
+                "examples": [
+                    {
+                        "any": [
+                            {
+                                "key": "solver_type",
+                                "pattern": "FEM",
+                            },
+                            {
+                                "key": "mesh_cells",
+                                "pattern": "1*",
+                            },
+                        ]
+                    },
+                    {
+                        "any": [
+                            {
+                                "key": "solver_type",
+                                "pattern": "*CFD*",
+                            }
+                        ]
+                    },
+                ]
+            }
+        )
+
+    model_config = ConfigDict(
+        json_schema_extra=_update_json_schema_extra,
+    )

services/api-server/tests/unit/test_api_dependencies.py

@@ -0,0 +1,146 @@
+from typing import Annotated
+
+import pytest
+from fastapi import Depends, FastAPI, status
+from fastapi.testclient import TestClient
+from pydantic import ValidationError
+from simcore_service_api_server.api.dependencies.models_schemas_job_filters import (
+    get_job_metadata_filter,
+)
+from simcore_service_api_server.models.schemas.jobs_filters import (
+    JobMetadataFilter,
+    MetadataFilterItem,
+)
+
+
+def test_get_metadata_filter():
+    # Test with None input
+    assert get_job_metadata_filter(None) is None
+
+    # Test with empty list
+    assert get_job_metadata_filter([]) is None
+
+    # Test with valid input (matching the example in the docstring)
+    input_data = ["key1:val*", "key2:exactval"]
+    result = get_job_metadata_filter(input_data)
+
+    expected = JobMetadataFilter(
+        any=[
+            MetadataFilterItem(name="key1", pattern="val*"),
+            MetadataFilterItem(name="key2", pattern="exactval"),
+        ]
+    )
+
+    assert result is not None
+    assert len(result.any) == 2
+    assert result.any[0].name == "key1"
+    assert result.any[0].pattern == "val*"
+    assert result.any[1].name == "key2"
+    assert result.any[1].pattern == "exactval"
+    assert result == expected
+
+    # Test with invalid input (missing colon)
+    input_data = ["key1val", "key2:exactval"]
+    result = get_job_metadata_filter(input_data)
+
+    assert result is not None
+    assert len(result.any) == 1
+    assert result.any[0].name == "key2"
+    assert result.any[0].pattern == "exactval"
+
+    # Test with empty pattern not allowed
+    input_data = ["key1:", "key2:exactval"]
+    with pytest.raises(ValidationError) as exc_info:
+        get_job_metadata_filter(input_data)
+
+    assert exc_info.value.errors()[0]["type"] == "string_too_short"
+
+
+def test_metadata_filter_in_api_route():
+    # Create a test FastAPI app
+    app = FastAPI()
+
+    # Define a route that uses the get_metadata_filter dependency
+    @app.get("/test-filter")
+    def filter_endpoint(
+        metadata_filter: Annotated[
+            JobMetadataFilter | None, Depends(get_job_metadata_filter)
+        ] = None,
+    ):
+        if not metadata_filter:
+            return {"filters": None}
+
+        # Convert to dict for easier comparison in test
+        return {
+            "filters": {
+                "any": [
+                    {"name": item.name, "pattern": item.pattern}
+                    for item in metadata_filter.any
+                ]
+            }
+        }
+
+    # Create a test client
+    client = TestClient(app)
+
+    # Test with no filter
+    response = client.get("/test-filter")
+    assert response.status_code == status.HTTP_200_OK
+    assert response.json() == {"filters": None}
+
+    # Test with single filter
+    response = client.get("/test-filter?metadata.any=key1:val*")
+    assert response.status_code == status.HTTP_200_OK
+    assert response.json() == {
+        "filters": {"any": [{"name": "key1", "pattern": "val*"}]}
+    }
+
+    # Test with multiple filters
+    response = client.get(
+        "/test-filter?metadata.any=key1:val*&metadata.any=key2:exactval"
+    )
+    assert response.status_code == status.HTTP_200_OK
+    assert response.json() == {
+        "filters": {
+            "any": [
+                {"name": "key1", "pattern": "val*"},
+                {"name": "key2", "pattern": "exactval"},
+            ]
+        }
+    }
+
+    # Test with invalid filter (should skip the invalid one)
+    response = client.get(
+        "/test-filter?metadata.any=invalid&metadata.any=key2:exactval"
+    )
+    assert response.status_code == status.HTTP_200_OK
+    assert response.json() == {
+        "filters": {"any": [{"name": "key2", "pattern": "exactval"}]}
+    }
+
+    # Test with URL-encoded characters
+    # Use special characters that need encoding: space, &, =, +, /, ?
+    encoded_query = "/test-filter?metadata.any=special%20key:value%20with%20spaces&metadata.any=symbols:a%2Bb%3Dc%26d%3F%2F"
+    response = client.get(encoded_query)
+    assert response.status_code == status.HTTP_200_OK
+    assert response.json() == {
+        "filters": {
+            "any": [
+                {"name": "special key", "pattern": "value with spaces"},
+                {"name": "symbols", "pattern": "a+b=c&d?/"},
+            ]
+        }
+    }
+
+    # Test with Unicode characters
+    unicode_query = "/test-filter?metadata.any=emoji:%F0%9F%98%8A&metadata.any=international:caf%C3%A9"
+    response = client.get(unicode_query)
+    assert response.status_code == status.HTTP_200_OK
+    assert response.json() == {
+        "filters": {
+            "any": [
+                {"name": "emoji", "pattern": "😊"},
+                {"name": "international", "pattern": "café"},
+            ]
+        }
+    }