Merge branch 'main' into CAT-1413

Author: YuriZmytrakov

CHANGELOG.md

@@ -10,9 +10,13 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ### Added
 
+- Added the `ENV_MAX_LIMIT` environment variable to SFEOS, allowing overriding of the `MAX_LIMIT`, which controls the `?limit` parameter for returned items and STAC collections. [#434](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/434)
+- Updated the `format_datetime_range` function to support milliseconds. [#423](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/423)
+
 ### Changed
 
 - Changed assets serialization to prevent mapping explosion while allowing asset inforamtion to be indexed. [#341](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/341)
+- Blocked the /collections/{collection_id}/bulk_items endpoint when environmental variable ENABLE_DATETIME_INDEX_FILTERING is set to true. [#438](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/438)
 
 ### Fixed
 
@@ -22,7 +26,6 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 - Added `id` field as secondary sort to sort config to ensure unique pagination tokens. [#421](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/421)
 - Added default environment variable `STAC_ITEM_LIMIT` to SFEOS for result limiting of returned items and STAC collections [#419](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/419)
-- Updated the `format_datetime_range` function to support milliseconds. [#423](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/423)
 
 ### Changed
 

README.md

@@ -228,6 +228,7 @@ You can customize additional settings in your `.env` file:
 | `DATABASE_REFRESH`           | Controls whether database operations refresh the index immediately after changes. If set to `true`, changes will be immediately searchable. If set to `false`, changes may not be immediately visible but can improve performance for bulk operations. If set to `wait_for`, changes will wait for the next refresh cycle to become visible. | `false` | Optional |
 | `ENABLE_TRANSACTIONS_EXTENSIONS` | Enables or disables the Transactions and Bulk Transactions API extensions. If set to `false`, the POST `/collections` route and related transaction endpoints (including bulk transaction operations) will be unavailable in the API. This is useful for deployments where mutating the catalog via the API should be prevented. | `true` | Optional |
 | `STAC_ITEM_LIMIT` | Sets the environment variable for result limiting to SFEOS for the number of returned items and STAC collections. | `10` | Optional |
+| `ENV_MAX_LIMIT` | Configures the environment variable in SFEOS to override the default `MAX_LIMIT`, which controls the limit parameter for returned items and STAC collections. | `10,000` | Optional |
 
 > [!NOTE]
 > The variables `ES_HOST`, `ES_PORT`, `ES_USE_SSL`, `ES_VERIFY_CERTS` and `ES_TIMEOUT` apply to both Elasticsearch and OpenSearch backends, so there is no need to rename the key names to `OS_` even if you're using OpenSearch.

stac_fastapi/core/stac_fastapi/core/core.py

@@ -965,6 +965,13 @@ def bulk_item_insert(
             A string indicating the number of items successfully added.
         """
         request = kwargs.get("request")
+
+        if os.getenv("ENABLE_DATETIME_INDEX_FILTERING"):
+            raise HTTPException(
+                status_code=400,
+                detail="The /collections/{collection_id}/bulk_items endpoint is invalid when ENABLE_DATETIME_INDEX_FILTERING is set to true. Try using the /collections/{collection_id}/items endpoint.",
+            )
+
         if request:
             base_url = str(request.base_url)
         else:

stac_fastapi/core/stac_fastapi/core/utilities.py

@@ -10,7 +10,15 @@
 
 from stac_fastapi.types.stac import Item
 
-MAX_LIMIT = 10000
+
+def get_max_limit():
+    """
+    Retrieve a MAX_LIMIT value from an environment variable.
+
+    Returns:
+        int: The int value parsed from the environment variable.
+    """
+    return int(os.getenv("ENV_MAX_LIMIT", 10000))
 
 
 def get_bool_env(name: str, default: Union[bool, str] = False) -> bool:

stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py

@@ -17,7 +17,7 @@
 
 from stac_fastapi.core.base_database_logic import BaseDatabaseLogic
 from stac_fastapi.core.serializers import CollectionSerializer, ItemSerializer
-from stac_fastapi.core.utilities import MAX_LIMIT, bbox2polygon
+from stac_fastapi.core.utilities import bbox2polygon, get_max_limit
 from stac_fastapi.elasticsearch.config import AsyncElasticsearchSettings
 from stac_fastapi.elasticsearch.config import (
     ElasticsearchSettings as SyncElasticsearchSettings,
@@ -543,7 +543,7 @@ async def execute_search(
             index_param = ITEM_INDICES
             query = add_collections_to_body(collection_ids, query)
 
-        max_result_window = MAX_LIMIT
+        max_result_window = get_max_limit()
 
         size_limit = min(limit + 1, max_result_window)
 

stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py

@@ -15,7 +15,7 @@
 
 from stac_fastapi.core.base_database_logic import BaseDatabaseLogic
 from stac_fastapi.core.serializers import CollectionSerializer, ItemSerializer
-from stac_fastapi.core.utilities import MAX_LIMIT, bbox2polygon, get_bool_env
+from stac_fastapi.core.utilities import bbox2polygon, get_bool_env, get_max_limit
 from stac_fastapi.extensions.core.transaction.request import (
     PartialCollection,
     PartialItem,
@@ -575,7 +575,7 @@ async def execute_search(
 
         search_body["sort"] = sort if sort else DEFAULT_SORT
 
-        max_result_window = MAX_LIMIT
+        max_result_window = get_max_limit()
 
         size_limit = min(limit + 1, max_result_window)
 

stac_fastapi/tests/api/test_api.py

@@ -1540,3 +1540,27 @@ async def test_collection_items_limit_env_variable(
     assert resp.status_code == 200
     resp_json = resp.json()
     assert int(limit) == len(resp_json["features"])
+
+
+@pytest.mark.asyncio
+async def test_search_max_item_limit(
+    app_client, load_test_data, txn_client, monkeypatch
+):
+    limit = "10"
+    monkeypatch.setenv("ENV_MAX_LIMIT", limit)
+
+    test_collection = load_test_data("test_collection.json")
+    await create_collection(txn_client, test_collection)
+
+    item = load_test_data("test_item.json")
+
+    for i in range(20):
+        test_item = item.copy()
+        test_item["id"] = f"test-item-collection-{i}"
+        await create_item(txn_client, test_item)
+
+    resp = await app_client.get("/search", params={"limit": 20})
+
+    assert resp.status_code == 200
+    resp_json = resp.json()
+    assert int(limit) == len(resp_json["features"])