Merge branch 'main' into sort-item-collection

Author: jonhealy1

CHANGELOG.md

@@ -10,14 +10,15 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ### Added
 
-- Sort extension and sortby functionality to the item collection route. [#437](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/437)
-- Query extension and query functionality to the item collection route. [#437](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/437)
-- Filter extension and filter functionality to the item collection route. [#437](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/437)
+- Added the `ENV_MAX_LIMIT` environment variable to SFEOS, allowing overriding of the `MAX_LIMIT`, which controls the `?limit` parameter for returned items and STAC collections. [#434](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/434)
+- Sort, Query, and Filter extension and functionality to the item collection route. [#437](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/437)
 
 ### Changed
 
 - Changed assets serialization to prevent mapping explosion while allowing asset information to be indexed. [#341](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/341)
 - Simplified the item_collection function in core.py, moving the request to the get_search function. [#437](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/437)
+- Updated the `format_datetime_range` function to support milliseconds. [#423](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/423)
+- Blocked the /collections/{collection_id}/bulk_items endpoint when environmental variable ENABLE_DATETIME_INDEX_FILTERING is set to true. [#438](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/438)
 
 ### Fixed
 

README.md

@@ -228,6 +228,7 @@ You can customize additional settings in your `.env` file:
 | `DATABASE_REFRESH`           | Controls whether database operations refresh the index immediately after changes. If set to `true`, changes will be immediately searchable. If set to `false`, changes may not be immediately visible but can improve performance for bulk operations. If set to `wait_for`, changes will wait for the next refresh cycle to become visible. | `false` | Optional |
 | `ENABLE_TRANSACTIONS_EXTENSIONS` | Enables or disables the Transactions and Bulk Transactions API extensions. If set to `false`, the POST `/collections` route and related transaction endpoints (including bulk transaction operations) will be unavailable in the API. This is useful for deployments where mutating the catalog via the API should be prevented. | `true` | Optional |
 | `STAC_ITEM_LIMIT` | Sets the environment variable for result limiting to SFEOS for the number of returned items and STAC collections. | `10` | Optional |
+| `ENV_MAX_LIMIT` | Configures the environment variable in SFEOS to override the default `MAX_LIMIT`, which controls the limit parameter for returned items and STAC collections. | `10,000` | Optional |
 
 > [!NOTE]
 > The variables `ES_HOST`, `ES_PORT`, `ES_USE_SSL`, `ES_VERIFY_CERTS` and `ES_TIMEOUT` apply to both Elasticsearch and OpenSearch backends, so there is no need to rename the key names to `OS_` even if you're using OpenSearch.

stac_fastapi/core/stac_fastapi/core/core.py

@@ -959,6 +959,13 @@ def bulk_item_insert(
             A string indicating the number of items successfully added.
         """
         request = kwargs.get("request")
+
+        if os.getenv("ENABLE_DATETIME_INDEX_FILTERING"):
+            raise HTTPException(
+                status_code=400,
+                detail="The /collections/{collection_id}/bulk_items endpoint is invalid when ENABLE_DATETIME_INDEX_FILTERING is set to true. Try using the /collections/{collection_id}/items endpoint.",
+            )
+
         if request:
             base_url = str(request.base_url)
         else:

stac_fastapi/core/stac_fastapi/core/datetime_utils.py

@@ -17,17 +17,20 @@ def format_datetime_range(date_str: str) -> str:
     """
 
     def normalize(dt):
+        """Normalize datetime string and preserve millisecond precision."""
         dt = dt.strip()
         if not dt or dt == "..":
             return ".."
         dt_obj = rfc3339_str_to_datetime(dt)
         dt_utc = dt_obj.astimezone(timezone.utc)
-        return dt_utc.strftime("%Y-%m-%dT%H:%M:%SZ")
+        return dt_utc.isoformat(timespec="milliseconds").replace("+00:00", "Z")
 
     if not isinstance(date_str, str):
         return "../.."
+
     if "/" not in date_str:
         return f"{normalize(date_str)}/{normalize(date_str)}"
+
     try:
         start, end = date_str.split("/", 1)
     except Exception:

stac_fastapi/core/stac_fastapi/core/utilities.py

@@ -10,7 +10,15 @@
 
 from stac_fastapi.types.stac import Item
 
-MAX_LIMIT = 10000
+
+def get_max_limit():
+    """
+    Retrieve a MAX_LIMIT value from an environment variable.
+
+    Returns:
+        int: The int value parsed from the environment variable.
+    """
+    return int(os.getenv("ENV_MAX_LIMIT", 10000))
 
 
 def get_bool_env(name: str, default: Union[bool, str] = False) -> bool:

stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py

@@ -17,7 +17,7 @@
 
 from stac_fastapi.core.base_database_logic import BaseDatabaseLogic
 from stac_fastapi.core.serializers import CollectionSerializer, ItemSerializer
-from stac_fastapi.core.utilities import MAX_LIMIT, bbox2polygon
+from stac_fastapi.core.utilities import bbox2polygon, get_max_limit
 from stac_fastapi.elasticsearch.config import AsyncElasticsearchSettings
 from stac_fastapi.elasticsearch.config import (
     ElasticsearchSettings as SyncElasticsearchSettings,
@@ -543,7 +543,7 @@ async def execute_search(
             index_param = ITEM_INDICES
             query = add_collections_to_body(collection_ids, query)
 
-        max_result_window = MAX_LIMIT
+        max_result_window = get_max_limit()
 
         size_limit = min(limit + 1, max_result_window)
 

stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py

@@ -16,7 +16,7 @@
 
 from stac_fastapi.core.base_database_logic import BaseDatabaseLogic
 from stac_fastapi.core.serializers import CollectionSerializer, ItemSerializer
-from stac_fastapi.core.utilities import MAX_LIMIT, bbox2polygon
+from stac_fastapi.core.utilities import bbox2polygon, get_max_limit
 from stac_fastapi.extensions.core.transaction.request import (
     PartialCollection,
     PartialItem,
@@ -540,7 +540,7 @@ async def execute_search(
 
         search_body["sort"] = sort if sort else DEFAULT_SORT
 
-        max_result_window = MAX_LIMIT
+        max_result_window = get_max_limit()
 
         size_limit = min(limit + 1, max_result_window)
 

stac_fastapi/tests/api/test_api.py

@@ -1698,3 +1698,26 @@ async def test_filter_by_nonexistent_id(app_client, ctx, txn_client):
     assert (
         len(resp_json["features"]) == 0
     ), f"Expected no items with ID {non_existent_id}, but found {len(resp_json['features'])} matches"
+
+    
+async def test_search_max_item_limit(
+    app_client, load_test_data, txn_client, monkeypatch
+):
+    limit = "10"
+    monkeypatch.setenv("ENV_MAX_LIMIT", limit)
+
+    test_collection = load_test_data("test_collection.json")
+    await create_collection(txn_client, test_collection)
+
+    item = load_test_data("test_item.json")
+
+    for i in range(20):
+        test_item = item.copy()
+        test_item["id"] = f"test-item-collection-{i}"
+        await create_item(txn_client, test_item)
+
+    resp = await app_client.get("/search", params={"limit": 20})
+
+    assert resp.status_code == 200
+    resp_json = resp.json()
+    assert int(limit) == len(resp_json["features"])