fix: last modified filter

Author: giancarloromeo

packages/models-library/src/models_library/api_schemas_webserver/storage.py

@@ -1,12 +1,13 @@
+import datetime
 from pathlib import Path
-from typing import Annotated, Final
+from typing import Annotated, Final, Self
 
 from models_library.utils.common_validators import (
     MIN_NON_WILDCARD_CHARS,
     WILDCARD_CHARS,
     ensure_pattern_has_enough_characters,
 )
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, model_validator
 
 from ..api_schemas_storage.storage_schemas import (
     DEFAULT_NUMBER_OF_PATHS_PER_PAGE,
@@ -16,8 +17,8 @@
 from ..rest_pagination import CursorQueryParameters
 from ._base import InputSchema
 
-MAX_SEARCH_ITEMS_PER_PAGE: Final[int] = 25
-DEFAULT_MAX_SEARCH_ITEMS_PER_PAGE: Final[int] = 50
+MAX_SEARCH_ITEMS_PER_PAGE: Final[int] = 50
+DEFAULT_MAX_SEARCH_ITEMS_PER_PAGE: Final[int] = 25
 
 
 class StorageLocationPathParams(BaseModel):
@@ -53,18 +54,44 @@ class DataExportPost(InputSchema):
 
 
 class SearchBodyParams(InputSchema):
-    name_pattern: Annotated[
+    filename_pattern: Annotated[
         str,
         ensure_pattern_has_enough_characters(),
         Field(
-            description=f"Name pattern with wildcard support {tuple(WILDCARD_CHARS)}. Minimum of {MIN_NON_WILDCARD_CHARS} non-wildcard characters required.",
+            description=f"File name pattern with wildcard support {tuple(WILDCARD_CHARS)}. Minimum of {MIN_NON_WILDCARD_CHARS} non-wildcard characters required.",
         ),
     ]
-    max_items_per_page: Annotated[
+    last_modified_before: Annotated[
+        datetime.datetime | None,
+        Field(
+            default=None,
+            description="Filter results to files modified before this date (inclusive). Format: YYYY-MM-DDTHH:MM:SS",
+        ),
+    ]
+    last_modified_after: Annotated[
+        datetime.datetime | None,
+        Field(
+            default=None,
+            description="Filter results to files modified after this date (inclusive). Format: YYYY-MM-DDTHH:MM:SS",
+        ),
+    ]
+    items_per_page: Annotated[
         int,
         Field(
-            description="Max number of items per page",
+            description="Number of items per page",
             ge=1,
             le=MAX_SEARCH_ITEMS_PER_PAGE,
         ),
     ] = DEFAULT_MAX_SEARCH_ITEMS_PER_PAGE
+
+    @model_validator(mode="after")
+    def _validate_date_range(self) -> Self:
+        """Ensure that last_modified_before is after last_modified_after when both are present."""
+        if (
+            self.last_modified_before is not None
+            and self.last_modified_after is not None
+            and self.last_modified_before <= self.last_modified_after
+        ):
+            msg = "last_modified_before must be after last_modified_after"
+            raise ValueError(msg)
+        return self

packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/storage/simcore_s3.py

@@ -1,3 +1,4 @@
+import datetime
 from typing import Literal
 
 from models_library.api_schemas_rpc_async_jobs.async_jobs import (
@@ -51,15 +52,19 @@ async def start_search(
     rabbitmq_rpc_client: RabbitMQRPCClient,
     *,
     job_filter: AsyncJobFilter,
-    name_pattern: str,
-    max_items_per_page: int,
+    items_per_page: int,
+    filename_pattern: str,
+    last_modified_before: datetime.datetime | None = None,
+    last_modified_after: datetime.datetime | None = None,
 ) -> tuple[AsyncJobGet, AsyncJobFilter]:
     async_job_rpc_get = await submit(
         rabbitmq_rpc_client,
         rpc_namespace=STORAGE_RPC_NAMESPACE,
-        method_name=TypeAdapter(RPCMethodName).validate_python("start_search"),
+        method_name=TypeAdapter(RPCMethodName).validate_python("start_file_search"),
         job_filter=job_filter,
-        name_pattern=name_pattern,
-        max_items_per_page=max_items_per_page,
+        items_per_page=items_per_page,
+        name_pattern=filename_pattern,
+        last_modified_before=last_modified_before,
+        last_modified_after=last_modified_after,
     )
     return async_job_rpc_get, job_filter

services/storage/src/simcore_service_storage/api/_worker_tasks/_simcore_s3.py

@@ -1,3 +1,4 @@
+import datetime
 import functools
 import logging
 from typing import Any
@@ -139,7 +140,9 @@ async def search(
     user_id: UserID,
     project_id: ProjectID | None,
     name_pattern: str,
-    max_items_per_page: int,
+    modified_before: datetime.datetime | None,
+    modified_after: datetime.datetime | None,
+    items_per_page: int,
 ) -> None:
     with log_context(
         _logger,
@@ -157,7 +160,9 @@ async def search(
             user_id=user_id,
             project_id=project_id,
             name_pattern=name_pattern,
-            items_per_page=max_items_per_page,
+            modified_before=modified_before,
+            modified_after=modified_after,
+            items_per_page=items_per_page,
         ):
             data = [
                 SearchResult(

services/storage/src/simcore_service_storage/api/rpc/_simcore_s3.py

@@ -72,7 +72,7 @@ async def start_search(
     task_manager: TaskManager,
     job_filter: AsyncJobFilter,
     name_pattern: str,
-    max_items_per_page: int,
+    items_per_page: int,
     project_id: str | None = None,
 ) -> AsyncJobGet:
     task_name = search.__name__
@@ -85,6 +85,6 @@ async def start_search(
         user_id=job_filter.user_id,
         project_id=project_id,
         name_pattern=name_pattern,
-        max_items_per_page=max_items_per_page,
+        items_per_page=items_per_page,
     )
     return AsyncJobGet(job_id=task_uuid, job_name=task_name)

services/storage/src/simcore_service_storage/simcore_s3_dsm.py

@@ -989,58 +989,60 @@ async def _process_s3_page_results(
 
     async def _search_project_s3_files(
         self,
+        user_id: UserID,
         proj_id: ProjectID,
         filename_pattern: str,
-        user_id: UserID,
-        items_per_page: NonNegativeInt,
-    ) -> AsyncGenerator[list[FileMetaData], None]:
-        """Search S3 files in a specific project and yield results page by page."""
+        last_modified_before: datetime.datetime | None = None,
+        last_modified_after: datetime.datetime | None = None,
+    ) -> AsyncGenerator[FileMetaData, None]:
+        """Search S3 files in a specific project and yield individual results."""
         s3_client = get_s3_client(self.app)
         min_parts_for_valid_s3_object = 2
-        current_page_results: list[FileMetaData] = []
 
         try:
             async for s3_objects in s3_client.list_objects_paginated(
                 bucket=self.simcore_bucket_name,
                 prefix=f"{proj_id}/",
-                items_per_page=items_per_page * 5,  # fetch more to filter locally
+                items_per_page=500,  # fetch larger batches for efficiency
             ):
                 for s3_obj in s3_objects:
                     filename = Path(s3_obj.object_key).name
 
-                    if (
+                    if not (
                         fnmatch.fnmatch(filename, filename_pattern)
                         and len(s3_obj.object_key.split("/"))
                         >= min_parts_for_valid_s3_object
                     ):
-                        file_meta = FileMetaData.from_simcore_node(
-                            user_id=user_id,
-                            file_id=TypeAdapter(SimcoreS3FileID).validate_python(
-                                s3_obj.object_key
-                            ),
-                            bucket=self.simcore_bucket_name,
-                            location_id=self.get_location_id(),
-                            location_name=self.get_location_name(),
-                            sha256_checksum=None,
-                            file_size=s3_obj.size,
-                            last_modified=s3_obj.last_modified,
-                            entity_tag=s3_obj.e_tag,
-                        )
-                        current_page_results.append(file_meta)
+                        continue
 
-                        if len(current_page_results) >= items_per_page:
-                            processed_results = await self._process_s3_page_results(
-                                current_page_results[:items_per_page]
-                            )
-                            yield processed_results
-                            current_page_results = current_page_results[items_per_page:]
+                    if (
+                        last_modified_before
+                        and s3_obj.last_modified
+                        and s3_obj.last_modified >= last_modified_before
+                    ):
+                        continue
+
+                    if (
+                        last_modified_after
+                        and s3_obj.last_modified
+                        and s3_obj.last_modified <= last_modified_after
+                    ):
+                        continue
 
-            # Handle remaining results, ensuring we don't exceed items_per_page
-            while current_page_results:
-                batch = current_page_results[:items_per_page]
-                current_page_results = current_page_results[items_per_page:]
-                processed_results = await self._process_s3_page_results(batch)
-                yield processed_results
+                    file_meta = FileMetaData.from_simcore_node(
+                        user_id=user_id,
+                        file_id=TypeAdapter(SimcoreS3FileID).validate_python(
+                            s3_obj.object_key
+                        ),
+                        bucket=self.simcore_bucket_name,
+                        location_id=self.get_location_id(),
+                        location_name=self.get_location_name(),
+                        sha256_checksum=None,
+                        file_size=s3_obj.size,
+                        last_modified=s3_obj.last_modified,
+                        entity_tag=s3_obj.e_tag,
+                    )
+                    yield file_meta
 
         except S3KeyNotFoundError:
             with log_context(
@@ -1054,6 +1056,8 @@ async def search(
         *,
         name_pattern: str,
         project_id: ProjectID | None = None,
+        modified_before: datetime.datetime | None = None,
+        modified_after: datetime.datetime | None = None,
         items_per_page: NonNegativeInt = 100,
     ) -> AsyncGenerator[list[FileMetaData], None]:
         """
@@ -1064,22 +1068,42 @@ async def search(
             user_id: The user requesting the search
             name_pattern: Wildcard pattern for filename matching (e.g., "*.txt", "test_*.json")
             project_id: Optional project ID to limit search to specific project
+            modified_before: Optional datetime filter - only include files modified before this datetime
+            modified_after: Optional datetime filter - only include files modified after this datetime
             items_per_page: Number of items to return per page
 
         Yields:
-            List of FileMetaData objects for each page
+            List of FileMetaData objects for each page, with exactly items_per_page items
+            (except the last page which may have fewer)
         """
         # Validate access rights
         accessible_projects_ids = await get_accessible_project_ids(
             get_db_engine(self.app), user_id=user_id, project_id=project_id
         )
 
-        # Search each accessible project
+        # Collect all results across projects
+        current_page_results: list[FileMetaData] = []
+
         for proj_id in accessible_projects_ids:
-            async for page_results in self._search_project_s3_files(
-                proj_id, name_pattern, user_id, items_per_page
+            async for file_result in self._search_project_s3_files(
+                user_id, proj_id, name_pattern, modified_before, modified_after
             ):
-                yield page_results
+                current_page_results.append(file_result)
+
+                if len(current_page_results) >= items_per_page:
+                    page_batch = current_page_results[:items_per_page]
+                    remaining_results = current_page_results[items_per_page:]
+
+                    processed_page = await self._process_s3_page_results(page_batch)
+                    yield processed_page
+
+                    # NOTE: keep the remaining results for next page
+                    current_page_results = remaining_results
+
+        # Handle any remaining results (the last page)
+        if current_page_results:
+            processed_page = await self._process_s3_page_results(current_page_results)
+            yield processed_page
 
     async def create_soft_link(
         self, user_id: int, target_file_id: StorageFileID, link_file_id: StorageFileID

services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml

@@ -17106,21 +17106,37 @@ components:
         \ - Task was aborted before completion"
     SearchBodyParams:
       properties:
-        namePattern:
+        filenamePattern:
           type: string
-          title: Namepattern
-          description: Name pattern with wildcard support ('*', '?'). Minimum of 3
-            non-wildcard characters required.
-        maxItemsPerPage:
+          title: Filenamepattern
+          description: File name pattern with wildcard support ('?', '*'). Minimum
+            of 3 non-wildcard characters required.
+        lastModifiedBefore:
+          anyOf:
+          - type: string
+            format: date-time
+          - type: 'null'
+          title: Lastmodifiedbefore
+          description: 'Filter results to files modified before this date (inclusive).
+            Format: YYYY-MM-DDTHH:MM:SS'
+        lastModifiedAfter:
+          anyOf:
+          - type: string
+            format: date-time
+          - type: 'null'
+          title: Lastmodifiedafter
+          description: 'Filter results to files modified after this date (inclusive).
+            Format: YYYY-MM-DDTHH:MM:SS'
+        itemsPerPage:
           type: integer
           maximum: 25
           minimum: 1
-          title: Maxitemsperpage
-          description: Max number of items per page
+          title: Itemsperpage
+          description: Number of items per page
           default: 50
       type: object
       required:
-      - namePattern
+      - filenamePattern
       title: SearchBodyParams
     SelectBox:
       properties:

services/web/server/src/simcore_service_webserver/storage/_rest.py

@@ -569,8 +569,10 @@ class _PathParams(BaseModel):
             user_id=_req_ctx.user_id,
             product_name=_req_ctx.product_name,
         ),
-        name_pattern=search_body.name_pattern,
-        max_items_per_page=search_body.max_items_per_page,
+        filename_pattern=search_body.filename_pattern,
+        last_modified_before=search_body.last_modified_before,
+        last_modified_after=search_body.last_modified_after,
+        items_per_page=search_body.items_per_page,
     )
     _job_id = f"{async_job_rpc_get.job_id}"
     return create_data_response(