Merge pull request #1010 from RS-PYTHON/feat/rspy-564-limit-the-limit

RSPY 564: limit the limit

Author: Padeanu

services/cadip/rs_server_cadip/api/cadip_search.py

@@ -474,7 +474,7 @@ def process_session_search(  # type: ignore # pylint: disable=too-many-arguments
     sortby: str,
     limit: Annotated[
         Union[int, None],
-        Query(gt=0, le=10000, default=1000, description="Pagination Limit"),
+        Query(gt=0, default=100, description="Pagination Limit"),
     ],
     page: Union[int, None] = 1,
 ) -> stac_pydantic.ItemCollection:
@@ -487,7 +487,7 @@ def process_session_search(  # type: ignore # pylint: disable=too-many-arguments
         request (Request): The request object (unused).
         station (str): CADIP station identifier (e.g., MTI, SGS, MPU, INU).
         queryables: Lists of queryables applicable to search op.
-        limit (int, optional): Maximum number of products to return. Beetween 0 and 10000, defaults to 1000.
+        limit (int, optional): Maximum number of products to return. Greater than 0, defaults to 100.
         sortby (str): Sort by +/-fieldName (ascending/descending).
         page (int): Page number to be displayed, defaults to first one.
     Returns:

services/common/rs_server_common/data_retrieval/eodag_provider.py

@@ -116,30 +116,32 @@ def __init__(self, config_file: Path, provider: str):  # type: ignore
 
     def _specific_search(self, **kwargs) -> Union[SearchResult, List]:
         """
-        Conducts a search for products within a specified time range.
+        Conducts a search for products using the specified OData arguments.
 
-        This private method interfaces with the client's search functionality,
-        retrieving products that fall within the given time range. The 'between'
-        parameter is expected to be a TimeRange object, encompassing start and end
-        timestamps. The method returns a dictionary of products keyed by their
-        respective identifiers.
+        This private method interfaces with the EODAG client's search functionality
+        to retrieve products that match the given search parameters. It handles
+        special cases such as `PublicationDate` and session ID lists while enforcing
+        pagination constraints as per provider limitations.
 
         Args:
-            date_time: An object representing search datetime, can be fixed or open/closed interval.
+            **kwargs: Arbitrary keyword arguments specifying search parameters,
+                including all queryables defined in the provider's configuration as OData arguments.
 
         Returns:
-            SearchResult: A dictionary where keys are product identifiers and
-                            values are EOProduct instances.
-
-        Note:
-            The time format of the 'between' parameter should be verified or formatted
-            appropriately before invoking this method. The method also assumes that the
-            client's search function is correctly set up to handle the provided time
-            range format.
+            Union[SearchResult, List]: A `SearchResult` object containing the matched products
+            or an empty list if no matches are found.
 
         Raises:
-            Exception: If the search encounters an error or fails, an exception is raised.
+            HTTPException: If a validation error occurs in the search query.
+            SearchProductFailed: If the search request fails due to request errors,
+                misconfiguration, or authentication issues.
+            ValueError: If authentication with EODAG fails.
+
+        Notes:
+            - Ensures compliance with provider-specific constraints, such as pagination limits.
+            - Logs encountered errors and provides detailed messages in case of failures.
         """
+
         mapped_search_args: Dict[str, Union[str, None]] = {}
         if session_id := kwargs.pop("SessionId", None):
             # Map session_id to the appropriate eodag parameter
@@ -157,11 +159,6 @@ def _specific_search(self, **kwargs) -> Union[SearchResult, List]:
                 value = ", ".join(f"'{p}'" for p in platform) if isinstance(platform, list) else f"'{platform}'"
                 mapped_search_args[key] = value
 
-            # TODO: check if it is the right way of doing this, looks very cumbersome to do it for every field
-            # Tempfix, will be updated, to dirrectly, to verify kwargs and then forward to search.
-            if retransfer := kwargs.pop("retransfer", None):
-                mapped_search_args["Retransfer"] = str(retransfer).lower()
-
         if date_time := kwargs.pop("PublicationDate", False):
             # Since now both for files and sessions, time interval is optional, map it if provided.
             fixed, start, end = [str(date) if date else None for date in date_time]
@@ -172,6 +169,14 @@ def _specific_search(self, **kwargs) -> Union[SearchResult, List]:
                     "StopPublicationDate": end,
                 },
             )
+        max_items_allowed = int(self.client.providers_config[self.provider].search.pagination["max_items_per_page"])
+        if int(kwargs["items_per_page"]) > max_items_allowed:
+            logger.warning(
+                f"Requesting {kwargs['items_per_page']} exceeds maximum of {max_items_allowed} "
+                "allowed for this provider!",
+            )
+            logger.warning(f"Number of items per page was set to {max_items_allowed - 1}.")
+            kwargs["items_per_page"] = max_items_allowed - 1
         try:
             logger.info(f"Searching from {self.provider} with parameters {mapped_search_args}")
             # Start search -> user defined search params in mapped_search_args (id), pagination in kwargs (top, limit).

tests/test_search_endpoint.py

@@ -903,6 +903,36 @@ def test_invalid_limit_values(self, client, endpoint):
             "Input should be greater than 0",
         )
 
+    @pytest.mark.unit
+    @pytest.mark.parametrize(
+        "endpoint, odata",
+        [
+            (
+                "/auxip/collections/s2_adgs2_AUX_OBMEMC/items?limit=10000000",
+                "http://127.0.0.1:5001/Products?$filter=Attributes/OData.CSC.StringAttribute/any(att:att/Name eq "
+                "'productType' and att/OData.CSC.StringAttribute/Value eq 'AUX_OBMEMC')&$orderby=PublicationDate "
+                "desc&$top=9999&$skip=0&$expand=Attributes",
+            ),
+            (
+                "/cadip/collections/cadip_session_by_id/items?limit=10000000",
+                "http://127.0.0.1:5000/Sessions?$filter=SessionId eq 'S1A_20200105072204051312'&$orderby="
+                "PublicationDate desc&$top=9999&$skip=0",
+            ),
+        ],
+    )
+    @responses.activate
+    def test_bigger_limit_than_allowed(self, client, mock_token_validation, endpoint, odata):
+        """
+        Test that if user request with a limit value bigger than max allowed in config
+        the limit value is set to max_allowed - 1.
+        Limit in request is set to 1_000_000, for given collection max allowed is set to 10000, therefore
+        in the final odata request, $top is set to 9999.
+        """
+        mock_token_validation()
+        responses.add(responses.GET, odata, json={"value": []}, status=200)
+        response = client.get(endpoint)
+        assert response.status_code == status.HTTP_200_OK
+
     @pytest.mark.unit
     @pytest.mark.parametrize(
         "endpoint",