From f98b9c27012724597f912c4b791b320c04e5c508 Mon Sep 17 00:00:00 2001 From: thomas loubrieu Date: Sun, 7 Dec 2025 16:29:31 -0800 Subject: [PATCH 1/5] make datetime fields constants in the DatabaseLogic --- .../stac_fastapi/opensearch/database_logic.py | 37 ++++++++++--------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py index 05aac1763..3987f2289 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py @@ -154,6 +154,11 @@ def __attrs_post_init__(self): aggregation_mapping: Dict[str, Dict[str, Any]] = AGGREGATION_MAPPING + # Private constant for the datetime property field + PROPERTIES_DATETIME_FIELD = "properties.datetime" + PROPERTIES_START_DATETIME_FIELD = "properties.start_datetime" + PROPERTIES_END_DATETIME_FIELD = "properties.end_datetime" + """CORE LOGIC""" async def get_all_collections( @@ -488,7 +493,7 @@ def apply_datetime_filter( Q( "bool", filter=[ - Q("exists", field="properties.datetime"), + Q("exists", field=DatabaseLogic.PROPERTIES_DATETIME_FIELD), Q( "term", **{"properties__datetime": datetime_search["eq"]}, @@ -497,10 +502,10 @@ def apply_datetime_filter( ), Q( "bool", - must_not=[Q("exists", field="properties.datetime")], + must_not=[Q("exists", field=DatabaseLogic.PROPERTIES_DATETIME_FIELD)], filter=[ - Q("exists", field="properties.start_datetime"), - Q("exists", field="properties.end_datetime"), + Q("exists", field=DatabaseLogic.PROPERTIES_START_DATETIME_FIELD), + Q("exists", field=DatabaseLogic.PROPERTIES_END_DATETIME_FIELD), Q( "range", properties__start_datetime={ @@ -522,7 +527,7 @@ def apply_datetime_filter( Q( "bool", filter=[ - Q("exists", field="properties.datetime"), + Q("exists", field=DatabaseLogic.PROPERTIES_DATETIME_FIELD), Q( "range", properties__datetime={ @@ -534,21 +539,17 @@ def apply_datetime_filter( ), Q( "bool", - must_not=[Q("exists", field="properties.datetime")], + must_not=[Q("exists", field=DatabaseLogic.PROPERTIES_DATETIME_FIELD)], filter=[ - Q("exists", field="properties.start_datetime"), - Q("exists", field="properties.end_datetime"), + Q("exists", field=DatabaseLogic.PROPERTIES_START_DATETIME_FIELD), + Q("exists", field=DatabaseLogic.PROPERTIES_END_DATETIME_FIELD), Q( "range", - properties__start_datetime={ - "lte": datetime_search["lte"] - }, + properties__start_datetime={"lte": datetime_search["lte"]}, ), Q( "range", - properties__end_datetime={ - "gte": datetime_search["gte"] - }, + properties__end_datetime={"gte": datetime_search["gte"]}, ), ], ), @@ -563,8 +564,8 @@ def apply_datetime_filter( filter_query = Q( "bool", filter=[ - Q("exists", field="properties.start_datetime"), - Q("exists", field="properties.end_datetime"), + Q("exists", field=DatabaseLogic.PROPERTIES_START_DATETIME_FIELD), + Q("exists", field=DatabaseLogic.PROPERTIES_END_DATETIME_FIELD), Q( "range", properties__start_datetime={"lte": datetime_search["eq"]}, @@ -579,8 +580,8 @@ def apply_datetime_filter( filter_query = Q( "bool", filter=[ - Q("exists", field="properties.start_datetime"), - Q("exists", field="properties.end_datetime"), + Q("exists", field=DatabaseLogic.PROPERTIES_START_DATETIME_FIELD), + Q("exists", field=DatabaseLogic.PROPERTIES_END_DATETIME_FIELD), Q( "range", properties__start_datetime={"lte": datetime_search["lte"]}, From 9268b46f067312105cf24c84353d63485680ebae Mon Sep 17 00:00:00 2001 From: thomas loubrieu Date: Mon, 8 Dec 2025 18:47:01 -0800 Subject: [PATCH 2/5] update obsolete comment --- .../stac_fastapi/opensearch/database_logic.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py index 3987f2289..f92385dd5 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py @@ -154,10 +154,14 @@ def __attrs_post_init__(self): aggregation_mapping: Dict[str, Dict[str, Any]] = AGGREGATION_MAPPING - # Private constant for the datetime property field + # constants for field names + # they are used in multiple methods + # and could be overwritten in subclasses used with alternate opensearch mappings. PROPERTIES_DATETIME_FIELD = "properties.datetime" PROPERTIES_START_DATETIME_FIELD = "properties.start_datetime" PROPERTIES_END_DATETIME_FIELD = "properties.end_datetime" + COLLECTION_FIELD = "collection" + GEOMETRY_FIELD = "geometry" """CORE LOGIC""" @@ -441,7 +445,7 @@ def apply_ids_filter(search: Search, item_ids: List[str]): @staticmethod def apply_collections_filter(search: Search, collection_ids: List[str]): """Database logic to search a list of STAC collection ids.""" - return search.filter("terms", collection=collection_ids) + return search.filter("terms", **{DatabaseLogic.COLLECTION_FIELD:collection_ids}) @staticmethod def apply_free_text_filter(search: Search, free_text_queries: Optional[List[str]]): @@ -613,7 +617,7 @@ def apply_bbox_filter(search: Search, bbox: List): Q( { "geo_shape": { - "geometry": { + DatabaseLogic.GEOMETRY_FIELD: { "shape": { "type": "polygon", "coordinates": bbox2polygon(*bbox), From 310a25d058ae19a463d34069ba8a1c0a2c8e05e6 Mon Sep 17 00:00:00 2001 From: thomas loubrieu Date: Mon, 8 Dec 2025 20:27:30 -0800 Subject: [PATCH 3/5] update CHANGELOG, lint fix --- CHANGELOG.md | 3 ++ .../stac_fastapi/opensearch/database_logic.py | 50 +++++++++++++++---- 2 files changed, 42 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 679f1ba62..546b30c3a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,8 +11,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Added catalogs route support to enable federated hierarchical catalog browsing and navigation in the STAC API. [#547](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/547) + ### Changed +- Have opensearch datetime, geometry and collections fields defined as constant strings [#553](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/553) + ### Fixed - Fix unawaited coroutine in `stac_fastapi.core.core`. [#551](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/551) diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py index 673c14ee1..3f5bf70d0 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py @@ -445,7 +445,9 @@ def apply_ids_filter(search: Search, item_ids: List[str]): @staticmethod def apply_collections_filter(search: Search, collection_ids: List[str]): """Database logic to search a list of STAC collection ids.""" - return search.filter("terms", **{DatabaseLogic.COLLECTION_FIELD:collection_ids}) + return search.filter( + "terms", **{DatabaseLogic.COLLECTION_FIELD: collection_ids} + ) @staticmethod def apply_free_text_filter(search: Search, free_text_queries: Optional[List[str]]): @@ -506,10 +508,18 @@ def apply_datetime_filter( ), Q( "bool", - must_not=[Q("exists", field=DatabaseLogic.PROPERTIES_DATETIME_FIELD)], + must_not=[ + Q("exists", field=DatabaseLogic.PROPERTIES_DATETIME_FIELD) + ], filter=[ - Q("exists", field=DatabaseLogic.PROPERTIES_START_DATETIME_FIELD), - Q("exists", field=DatabaseLogic.PROPERTIES_END_DATETIME_FIELD), + Q( + "exists", + field=DatabaseLogic.PROPERTIES_START_DATETIME_FIELD, + ), + Q( + "exists", + field=DatabaseLogic.PROPERTIES_END_DATETIME_FIELD, + ), Q( "range", properties__start_datetime={ @@ -543,17 +553,29 @@ def apply_datetime_filter( ), Q( "bool", - must_not=[Q("exists", field=DatabaseLogic.PROPERTIES_DATETIME_FIELD)], + must_not=[ + Q("exists", field=DatabaseLogic.PROPERTIES_DATETIME_FIELD) + ], filter=[ - Q("exists", field=DatabaseLogic.PROPERTIES_START_DATETIME_FIELD), - Q("exists", field=DatabaseLogic.PROPERTIES_END_DATETIME_FIELD), + Q( + "exists", + field=DatabaseLogic.PROPERTIES_START_DATETIME_FIELD, + ), + Q( + "exists", + field=DatabaseLogic.PROPERTIES_END_DATETIME_FIELD, + ), Q( "range", - properties__start_datetime={"lte": datetime_search["lte"]}, + properties__start_datetime={ + "lte": datetime_search["lte"] + }, ), Q( "range", - properties__end_datetime={"gte": datetime_search["gte"]}, + properties__end_datetime={ + "gte": datetime_search["gte"] + }, ), ], ), @@ -568,7 +590,10 @@ def apply_datetime_filter( filter_query = Q( "bool", filter=[ - Q("exists", field=DatabaseLogic.PROPERTIES_START_DATETIME_FIELD), + Q( + "exists", + field=DatabaseLogic.PROPERTIES_START_DATETIME_FIELD, + ), Q("exists", field=DatabaseLogic.PROPERTIES_END_DATETIME_FIELD), Q( "range", @@ -584,7 +609,10 @@ def apply_datetime_filter( filter_query = Q( "bool", filter=[ - Q("exists", field=DatabaseLogic.PROPERTIES_START_DATETIME_FIELD), + Q( + "exists", + field=DatabaseLogic.PROPERTIES_START_DATETIME_FIELD, + ), Q("exists", field=DatabaseLogic.PROPERTIES_END_DATETIME_FIELD), Q( "range", From c84db1607b95f84fe97a4017635a47788d3e7916 Mon Sep 17 00:00:00 2001 From: thomas loubrieu Date: Thu, 11 Dec 2025 19:31:18 -0800 Subject: [PATCH 4/5] fix forgotten nested properties management, extend changes to elasticsearch --- .../elasticsearch/database_logic.py | 137 ++++++++++++++---- .../stac_fastapi/opensearch/database_logic.py | 78 +++++++--- 2 files changed, 168 insertions(+), 47 deletions(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index 759156073..7576f2870 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -154,6 +154,20 @@ def __attrs_post_init__(self): aggregation_mapping: Dict[str, Dict[str, Any]] = AGGREGATION_MAPPING + # constants for field names + # they are used in multiple methods + # and could be overwritten in subclasses used with alternate opensearch mappings. + PROPERTIES_DATETIME_FIELD = "properties.datetime" + PROPERTIES_START_DATETIME_FIELD = "properties.start_datetime" + PROPERTIES_END_DATETIME_FIELD = "properties.end_datetime" + COLLECTION_FIELD = "collection" + GEOMETRY_FIELD = "geometry" + + @staticmethod + def __nested_field__(field: str): + """Convert opensearch field to nested field format.""" + return field.replace(".", "__") + """CORE LOGIC""" async def get_all_collections( @@ -436,7 +450,10 @@ def apply_ids_filter(search: Search, item_ids: List[str]): @staticmethod def apply_collections_filter(search: Search, collection_ids: List[str]): """Database logic to search a list of STAC collection ids.""" - return search.filter("terms", collection=collection_ids) + collection_nested_field = DatabaseLogic.__nested_field__( + DatabaseLogic.COLLECTION_FIELD + ) + return search.filter("terms", **{collection_nested_field: collection_ids}) @staticmethod def apply_datetime_filter( @@ -461,6 +478,16 @@ def apply_datetime_filter( if not datetime_search: return search, datetime_search + nested_datetime_field = DatabaseLogic.__nested_field__( + DatabaseLogic.PROPERTIES_DATETIME_FIELD + ) + nested_start_datetime_field = DatabaseLogic.__nested_field__( + DatabaseLogic.PROPERTIES_START_DATETIME_FIELD + ) + nested_end_datetime_field = DatabaseLogic.__nested_field__( + DatabaseLogic.PROPERTIES_END_DATETIME_FIELD + ) + if USE_DATETIME: if "eq" in datetime_search: # For exact matches, include: @@ -470,28 +497,42 @@ def apply_datetime_filter( Q( "bool", filter=[ - Q("exists", field="properties.datetime"), + Q("exists", field=DatabaseLogic.PROPERTIES_DATETIME_FIELD), Q( "term", - **{"properties__datetime": datetime_search["eq"]}, + **{nested_datetime_field: datetime_search["eq"]}, ), ], ), Q( "bool", - must_not=[Q("exists", field="properties.datetime")], + must_not=[ + Q("exists", field=DatabaseLogic.PROPERTIES_DATETIME_FIELD) + ], filter=[ - Q("exists", field="properties.start_datetime"), - Q("exists", field="properties.end_datetime"), + Q( + "exists", + field=DatabaseLogic.PROPERTIES_START_DATETIME_FIELD, + ), + Q( + "exists", + field=DatabaseLogic.PROPERTIES_END_DATETIME_FIELD, + ), Q( "range", - properties__start_datetime={ - "lte": datetime_search["eq"] + **{ + nested_start_datetime_field: { + "lte": datetime_search["eq"] + } }, ), Q( "range", - properties__end_datetime={"gte": datetime_search["eq"]}, + **{ + nested_end_datetime_field: { + "gte": datetime_search["eq"] + } + }, ), ], ), @@ -504,32 +545,46 @@ def apply_datetime_filter( Q( "bool", filter=[ - Q("exists", field="properties.datetime"), + Q("exists", field=DatabaseLogic.PROPERTIES_DATETIME_FIELD), Q( "range", - properties__datetime={ - "gte": datetime_search["gte"], - "lte": datetime_search["lte"], + **{ + nested_datetime_field: { + "gte": datetime_search["gte"], + "lte": datetime_search["lte"], + } }, ), ], ), Q( "bool", - must_not=[Q("exists", field="properties.datetime")], + must_not=[ + Q("exists", field=DatabaseLogic.PROPERTIES_DATETIME_FIELD) + ], filter=[ - Q("exists", field="properties.start_datetime"), - Q("exists", field="properties.end_datetime"), + Q( + "exists", + field=DatabaseLogic.PROPERTIES_START_DATETIME_FIELD, + ), + Q( + "exists", + field=DatabaseLogic.PROPERTIES_END_DATETIME_FIELD, + ), Q( "range", - properties__start_datetime={ - "lte": datetime_search["lte"] + **{ + nested_start_datetime_field: { + "lte": datetime_search["lte"] + } }, ), Q( "range", - properties__end_datetime={ - "gte": datetime_search["gte"] + **{ + nested_end_datetime_field: { + "gte": datetime_search["gte"] + } }, ), ], @@ -545,15 +600,26 @@ def apply_datetime_filter( filter_query = Q( "bool", filter=[ - Q("exists", field="properties.start_datetime"), - Q("exists", field="properties.end_datetime"), + Q( + "exists", + field=DatabaseLogic.PROPERTIES_START_DATETIME_FIELD, + ), + Q("exists", field=DatabaseLogic.PROPERTIES_END_DATETIME_FIELD), Q( "range", - properties__start_datetime={"lte": datetime_search["eq"]}, + **{ + nested_start_datetime_field: { + "lte": datetime_search["eq"] + } + }, ), Q( "range", - properties__end_datetime={"gte": datetime_search["eq"]}, + **{ + nested_end_datetime_field: { + "gte": datetime_search["eq"] + } + }, ), ], ) @@ -561,15 +627,26 @@ def apply_datetime_filter( filter_query = Q( "bool", filter=[ - Q("exists", field="properties.start_datetime"), - Q("exists", field="properties.end_datetime"), + Q( + "exists", + field=DatabaseLogic.PROPERTIES_START_DATETIME_FIELD, + ), + Q("exists", field=DatabaseLogic.PROPERTIES_END_DATETIME_FIELD), Q( "range", - properties__start_datetime={"lte": datetime_search["lte"]}, + **{ + nested_start_datetime_field: { + "lte": datetime_search["lte"] + } + }, ), Q( "range", - properties__end_datetime={"gte": datetime_search["gte"]}, + **{ + nested_end_datetime_field: { + "gte": datetime_search["gte"] + } + }, ), ], ) @@ -594,7 +671,7 @@ def apply_bbox_filter(search: Search, bbox: List): Q( { "geo_shape": { - "geometry": { + DatabaseLogic.GEOMETRY_FIELD: { "shape": { "type": "polygon", "coordinates": bbox2polygon(*bbox), @@ -1708,7 +1785,7 @@ def bulk_sync( kwargs = kwargs or {} # Resolve the `refresh` parameter - refresh = kwargs.get("refresh", self.async_settings.database_refresh) + refresh = kwargs.get("refresh", self.sync_settings.database_refresh) refresh = validate_refresh(refresh) # Log the bulk insert attempt diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py index 3f5bf70d0..1bf6c0c12 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py @@ -163,6 +163,11 @@ def __attrs_post_init__(self): COLLECTION_FIELD = "collection" GEOMETRY_FIELD = "geometry" + @staticmethod + def __nested_field__(field: str): + """Convert opensearch field to nested field format.""" + return field.replace(".", "__") + """CORE LOGIC""" async def get_all_collections( @@ -445,9 +450,10 @@ def apply_ids_filter(search: Search, item_ids: List[str]): @staticmethod def apply_collections_filter(search: Search, collection_ids: List[str]): """Database logic to search a list of STAC collection ids.""" - return search.filter( - "terms", **{DatabaseLogic.COLLECTION_FIELD: collection_ids} + collection_nested_field = DatabaseLogic.__nested_field__( + DatabaseLogic.COLLECTION_FIELD ) + return search.filter("terms", **{collection_nested_field: collection_ids}) @staticmethod def apply_free_text_filter(search: Search, free_text_queries: Optional[List[str]]): @@ -490,6 +496,16 @@ def apply_datetime_filter( # False: Always search only by start/end datetime USE_DATETIME = get_bool_env("USE_DATETIME", default=True) + nested_datetime_field = DatabaseLogic.__nested_field__( + DatabaseLogic.PROPERTIES_DATETIME_FIELD + ) + nested_start_datetime_field = DatabaseLogic.__nested_field__( + DatabaseLogic.PROPERTIES_START_DATETIME_FIELD + ) + nested_end_datetime_field = DatabaseLogic.__nested_field__( + DatabaseLogic.PROPERTIES_END_DATETIME_FIELD + ) + if USE_DATETIME: if "eq" in datetime_search: # For exact matches, include: @@ -502,7 +518,7 @@ def apply_datetime_filter( Q("exists", field=DatabaseLogic.PROPERTIES_DATETIME_FIELD), Q( "term", - **{"properties__datetime": datetime_search["eq"]}, + **{nested_datetime_field: datetime_search["eq"]}, ), ], ), @@ -522,13 +538,19 @@ def apply_datetime_filter( ), Q( "range", - properties__start_datetime={ - "lte": datetime_search["eq"] + **{ + nested_start_datetime_field: { + "lte": datetime_search["eq"] + } }, ), Q( "range", - properties__end_datetime={"gte": datetime_search["eq"]}, + **{ + nested_end_datetime_field: { + "gte": datetime_search["eq"] + } + }, ), ], ), @@ -544,9 +566,11 @@ def apply_datetime_filter( Q("exists", field=DatabaseLogic.PROPERTIES_DATETIME_FIELD), Q( "range", - properties__datetime={ - "gte": datetime_search["gte"], - "lte": datetime_search["lte"], + **{ + nested_datetime_field: { + "gte": datetime_search["gte"], + "lte": datetime_search["lte"], + } }, ), ], @@ -567,14 +591,18 @@ def apply_datetime_filter( ), Q( "range", - properties__start_datetime={ - "lte": datetime_search["lte"] + **{ + nested_start_datetime_field: { + "lte": datetime_search["lte"] + } }, ), Q( "range", - properties__end_datetime={ - "gte": datetime_search["gte"] + **{ + nested_end_datetime_field: { + "gte": datetime_search["gte"] + } }, ), ], @@ -597,11 +625,19 @@ def apply_datetime_filter( Q("exists", field=DatabaseLogic.PROPERTIES_END_DATETIME_FIELD), Q( "range", - properties__start_datetime={"lte": datetime_search["eq"]}, + **{ + nested_start_datetime_field: { + "lte": datetime_search["eq"] + } + }, ), Q( "range", - properties__end_datetime={"gte": datetime_search["eq"]}, + **{ + nested_end_datetime_field: { + "gte": datetime_search["eq"] + } + }, ), ], ) @@ -616,11 +652,19 @@ def apply_datetime_filter( Q("exists", field=DatabaseLogic.PROPERTIES_END_DATETIME_FIELD), Q( "range", - properties__start_datetime={"lte": datetime_search["lte"]}, + **{ + nested_start_datetime_field: { + "lte": datetime_search["lte"] + } + }, ), Q( "range", - properties__end_datetime={"gte": datetime_search["gte"]}, + **{ + nested_end_datetime_field: { + "gte": datetime_search["gte"] + } + }, ), ], ) From 2a43305d9c73be65ca2ba0c1c5478b606ff5006c Mon Sep 17 00:00:00 2001 From: thomas loubrieu Date: Sat, 13 Dec 2025 15:33:35 -0800 Subject: [PATCH 5/5] make filter fields environment variables, update the README accordingly --- README.md | 79 ++++++++++--------- .../elasticsearch/database_logic.py | 18 +++-- .../stac_fastapi/opensearch/database_logic.py | 18 +++-- 3 files changed, 66 insertions(+), 49 deletions(-) diff --git a/README.md b/README.md index 7744b7f41..249d08193 100644 --- a/README.md +++ b/README.md @@ -434,43 +434,48 @@ There are two main ways to run the API locally: You can customize additional settings in your `.env` file: -| Variable | Description | Default | Required | -|------------------------------|--------------------------------------------------------------------------------------|--------------------------|---------------------------------------------------------------------------------------------| -| `ES_HOST` | Hostname for external Elasticsearch/OpenSearch. | `localhost` | Optional | -| `ES_PORT` | Port for Elasticsearch/OpenSearch. | `9200` (ES) / `9202` (OS)| Optional | -| `ES_USE_SSL` | Use SSL for connecting to Elasticsearch/OpenSearch. | `true` | Optional | -| `ES_VERIFY_CERTS` | Verify SSL certificates when connecting. | `true` | Optional | -| `ES_API_KEY` | API Key for external Elasticsearch/OpenSearch. | N/A | Optional | -| `ES_TIMEOUT` | Client timeout for Elasticsearch/OpenSearch. | DB client default | Optional | -| `STAC_FASTAPI_TITLE` | Title of the API in the documentation. | `stac-fastapi-` | Optional | -| `STAC_FASTAPI_DESCRIPTION` | Description of the API in the documentation. | N/A | Optional | -| `STAC_FASTAPI_VERSION` | API version. | `2.1` | Optional | -| `STAC_FASTAPI_LANDING_PAGE_ID` | Landing page ID | `stac-fastapi` | Optional | -| `APP_HOST` | Server bind address. | `0.0.0.0` | Optional | -| `APP_PORT` | Server port. | `8000` | Optional | -| `ENVIRONMENT` | Runtime environment. | `local` | Optional | -| `WEB_CONCURRENCY` | Number of worker processes. | `10` | Optional | -| `RELOAD` | Enable auto-reload for development. | `true` | Optional | -| `STAC_FASTAPI_RATE_LIMIT` | API rate limit per client. | `200/minute` | Optional | -| `BACKEND` | Tests-related variable | `elasticsearch` or `opensearch` based on the backend | Optional | -| `ELASTICSEARCH_VERSION` | Version of Elasticsearch to use. | `8.11.0` | Optional | -| `OPENSEARCH_VERSION` | OpenSearch version | `2.11.1` | Optional | -| `ENABLE_DIRECT_RESPONSE` | Enable direct response for maximum performance (disables all FastAPI dependencies, including authentication, custom status codes, and validation) | `false` | Optional | -| `RAISE_ON_BULK_ERROR` | Controls whether bulk insert operations raise exceptions on errors. If set to `true`, the operation will stop and raise an exception when an error occurs. If set to `false`, errors will be logged, and the operation will continue. **Note:** STAC Item and ItemCollection validation errors will always raise, regardless of this flag. | `false` | Optional | -| `DATABASE_REFRESH` | Controls whether database operations refresh the index immediately after changes. If set to `true`, changes will be immediately searchable. If set to `false`, changes may not be immediately visible but can improve performance for bulk operations. If set to `wait_for`, changes will wait for the next refresh cycle to become visible. | `false` | Optional | -| `ENABLE_COLLECTIONS_SEARCH` | Enable collection search extensions (sort, fields, free text search, structured filtering, and datetime filtering) on the core `/collections` endpoint. | `true` | Optional | -| `ENABLE_COLLECTIONS_SEARCH_ROUTE` | Enable the custom `/collections-search` endpoint (both GET and POST methods). When disabled, the custom endpoint will not be available, but collection search extensions will still be available on the core `/collections` endpoint if `ENABLE_COLLECTIONS_SEARCH` is true. | `false` | Optional | -| `ENABLE_TRANSACTIONS_EXTENSIONS` | Enables or disables the Transactions and Bulk Transactions API extensions. This is useful for deployments where mutating the catalog via the API should be prevented. If set to `true`, the POST `/collections` route for search will be unavailable in the API. | `true` | Optional | -| `ENABLE_CATALOGS_ROUTE` | Enable the `/catalogs` endpoint for federated hierarchical catalog browsing and navigation. When enabled, provides access to federated STAC API architecture with hub-and-spoke pattern. | `false` | Optional | -| `STAC_GLOBAL_COLLECTION_MAX_LIMIT` | Configures the maximum number of STAC collections that can be returned in a single search request. | N/A | Optional | -| `STAC_DEFAULT_COLLECTION_LIMIT` | Configures the default number of STAC collections returned when no limit parameter is specified in the request. | `300` | Optional | -| `STAC_GLOBAL_ITEM_MAX_LIMIT` | Configures the maximum number of STAC items that can be returned in a single search request. | N/A | Optional | -| `STAC_DEFAULT_ITEM_LIMIT` | Configures the default number of STAC items returned when no limit parameter is specified in the request. | `10` | Optional | -| `STAC_INDEX_ASSETS` | Controls if Assets are indexed when added to Elasticsearch/Opensearch. This allows asset fields to be included in search queries. | `false` | Optional | -| `USE_DATETIME` | Configures the datetime search behavior in SFEOS. When enabled, searches both datetime field and falls back to start_datetime/end_datetime range for items with null datetime. When disabled, searches only by start_datetime/end_datetime range. | `true` | Optional | -| `USE_DATETIME_NANOS` | Enables nanosecond precision handling for `datetime` field searches as per the `date_nanos` type. When `False`, it uses 3 millisecond precision as per the type `date`. | `true` | Optional | -| `EXCLUDED_FROM_QUERYABLES` | Comma-separated list of fully qualified field names to exclude from the queryables endpoint and filtering. Use full paths like `properties.auth:schemes,properties.storage:schemes`. Excluded fields and their nested children will not be exposed in queryables. | None | Optional | -| `EXCLUDED_FROM_ITEMS` | Specifies fields to exclude from STAC item responses. Supports comma-separated field names and dot notation for nested fields (e.g., `private_data,properties.confidential,assets.internal`). | `None` | Optional | +| Variable | Description | Default | Required | +|----------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------| +| `ES_HOST` | Hostname for external Elasticsearch/OpenSearch. | `localhost` | Optional | +| `ES_PORT` | Port for Elasticsearch/OpenSearch. | `9200` (ES) / `9202` (OS) | Optional | +| `ES_USE_SSL` | Use SSL for connecting to Elasticsearch/OpenSearch. | `true` | Optional | +| `ES_VERIFY_CERTS` | Verify SSL certificates when connecting. | `true` | Optional | +| `ES_API_KEY` | API Key for external Elasticsearch/OpenSearch. | N/A | Optional | +| `ES_TIMEOUT` | Client timeout for Elasticsearch/OpenSearch. | DB client default | Optional | +| `STAC_FASTAPI_TITLE` | Title of the API in the documentation. | `stac-fastapi-` | Optional | +| `STAC_FASTAPI_DESCRIPTION` | Description of the API in the documentation. | N/A | Optional | +| `STAC_FASTAPI_VERSION` | API version. | `2.1` | Optional | +| `STAC_FASTAPI_LANDING_PAGE_ID` | Landing page ID | `stac-fastapi` | Optional | +| `APP_HOST` | Server bind address. | `0.0.0.0` | Optional | +| `APP_PORT` | Server port. | `8000` | Optional | +| `ENVIRONMENT` | Runtime environment. | `local` | Optional | +| `WEB_CONCURRENCY` | Number of worker processes. | `10` | Optional | +| `RELOAD` | Enable auto-reload for development. | `true` | Optional | +| `STAC_FASTAPI_RATE_LIMIT` | API rate limit per client. | `200/minute` | Optional | +| `BACKEND` | Tests-related variable | `elasticsearch` or `opensearch` based on the backend | Optional | +| `ELASTICSEARCH_VERSION` | Version of Elasticsearch to use. | `8.11.0` | Optional | +| `OPENSEARCH_VERSION` | OpenSearch version | `2.11.1` | Optional | +| `ENABLE_DIRECT_RESPONSE` | Enable direct response for maximum performance (disables all FastAPI dependencies, including authentication, custom status codes, and validation) | `false` | Optional | +| `RAISE_ON_BULK_ERROR` | Controls whether bulk insert operations raise exceptions on errors. If set to `true`, the operation will stop and raise an exception when an error occurs. If set to `false`, errors will be logged, and the operation will continue. **Note:** STAC Item and ItemCollection validation errors will always raise, regardless of this flag. | `false` | Optional | +| `DATABASE_REFRESH` | Controls whether database operations refresh the index immediately after changes. If set to `true`, changes will be immediately searchable. If set to `false`, changes may not be immediately visible but can improve performance for bulk operations. If set to `wait_for`, changes will wait for the next refresh cycle to become visible. | `false` | Optional | +| `ENABLE_COLLECTIONS_SEARCH` | Enable collection search extensions (sort, fields, free text search, structured filtering, and datetime filtering) on the core `/collections` endpoint. | `true` | Optional | +| `ENABLE_COLLECTIONS_SEARCH_ROUTE` | Enable the custom `/collections-search` endpoint (both GET and POST methods). When disabled, the custom endpoint will not be available, but collection search extensions will still be available on the core `/collections` endpoint if `ENABLE_COLLECTIONS_SEARCH` is true. | `false` | Optional | +| `ENABLE_TRANSACTIONS_EXTENSIONS` | Enables or disables the Transactions and Bulk Transactions API extensions. This is useful for deployments where mutating the catalog via the API should be prevented. If set to `true`, the POST `/collections` route for search will be unavailable in the API. | `true` | Optional | +| `ENABLE_CATALOGS_ROUTE` | Enable the `/catalogs` endpoint for federated hierarchical catalog browsing and navigation. When enabled, provides access to federated STAC API architecture with hub-and-spoke pattern. | `false` | Optional | +| `STAC_GLOBAL_COLLECTION_MAX_LIMIT` | Configures the maximum number of STAC collections that can be returned in a single search request. | N/A | Optional | +| `STAC_DEFAULT_COLLECTION_LIMIT` | Configures the default number of STAC collections returned when no limit parameter is specified in the request. | `300` | Optional | +| `STAC_GLOBAL_ITEM_MAX_LIMIT` | Configures the maximum number of STAC items that can be returned in a single search request. | N/A | Optional | +| `STAC_DEFAULT_ITEM_LIMIT` | Configures the default number of STAC items returned when no limit parameter is specified in the request. | `10` | Optional | +| `STAC_INDEX_ASSETS` | Controls if Assets are indexed when added to Elasticsearch/Opensearch. This allows asset fields to be included in search queries. | `false` | Optional | +| `USE_DATETIME` | Configures the datetime search behavior in SFEOS. When enabled, searches both datetime field and falls back to start_datetime/end_datetime range for items with null datetime. When disabled, searches only by start_datetime/end_datetime range. | `true` | Optional | +| `USE_DATETIME_NANOS` | Enables nanosecond precision handling for `datetime` field searches as per the `date_nanos` type. When `False`, it uses 3 millisecond precision as per the type `date`. | `true` | Optional | +| `EXCLUDED_FROM_QUERYABLES` | Comma-separated list of fully qualified field names to exclude from the queryables endpoint and filtering. Use full paths like `properties.auth:schemes,properties.storage:schemes`. Excluded fields and their nested children will not be exposed in queryables. | None | Optional | +| `EXCLUDED_FROM_ITEMS` | Specifies fields to exclude from STAC item responses. Supports comma-separated field names and dot notation for nested fields (e.g., `private_data,properties.confidential,assets.internal`). | `None` | Optional | +| `PROPERTIES_DATETIME_FIELD` | Specifies the field used for single datetime of the items in the backend database. | `properties.datetime` | Optional | +| `PROPERTIES_START_DATETIME_FIELD` | Specifies the field used for the lower value of a datetime range for the items in the backend database. | `properties.start_datetime` | Optional | +| `PROPERTIES_END_DATETIME_FIELD` | Specifies the field used for the upper value of a datetime range for the items in the backend database. | `properties.end_datetime` | Optional | +| `COLLECTION_FIELD` | Specifies the field used for the collection an item belongs to in the backend database | `collection` | Optional | +| `GEOMETRY_FIELD` | Specifies the field containing the geometry of the items in the backend database | `geometry` | Optional | > [!NOTE] diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index 7576f2870..c3eadc7ae 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -1,7 +1,7 @@ """Database logic.""" - import asyncio import logging +import os from base64 import urlsafe_b64decode, urlsafe_b64encode from copy import deepcopy from typing import Any, Dict, Iterable, List, Optional, Tuple, Type @@ -157,11 +157,17 @@ def __attrs_post_init__(self): # constants for field names # they are used in multiple methods # and could be overwritten in subclasses used with alternate opensearch mappings. - PROPERTIES_DATETIME_FIELD = "properties.datetime" - PROPERTIES_START_DATETIME_FIELD = "properties.start_datetime" - PROPERTIES_END_DATETIME_FIELD = "properties.end_datetime" - COLLECTION_FIELD = "collection" - GEOMETRY_FIELD = "geometry" + PROPERTIES_DATETIME_FIELD = os.getenv( + "STAC_FIELD_PROP_DATETIME", "properties.datetime" + ) + PROPERTIES_START_DATETIME_FIELD = os.getenv( + "STAC_FIELD_PROP_START_DATETIME", "properties.start_datetime" + ) + PROPERTIES_END_DATETIME_FIELD = os.getenv( + "STAC_FIELD_PROP_END_DATETIME", "properties.end_datetime" + ) + COLLECTION_FIELD = os.getenv("STAC_FIELD_COLLECTION", "collection") + GEOMETRY_FIELD = os.getenv("STAC_FIELD_GEOMETRY", "geometry") @staticmethod def __nested_field__(field: str): diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py index 1bf6c0c12..78d13d9c9 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py @@ -1,7 +1,7 @@ """Database logic.""" - import asyncio import logging +import os from base64 import urlsafe_b64decode, urlsafe_b64encode from collections.abc import Iterable from copy import deepcopy @@ -157,11 +157,17 @@ def __attrs_post_init__(self): # constants for field names # they are used in multiple methods # and could be overwritten in subclasses used with alternate opensearch mappings. - PROPERTIES_DATETIME_FIELD = "properties.datetime" - PROPERTIES_START_DATETIME_FIELD = "properties.start_datetime" - PROPERTIES_END_DATETIME_FIELD = "properties.end_datetime" - COLLECTION_FIELD = "collection" - GEOMETRY_FIELD = "geometry" + PROPERTIES_DATETIME_FIELD = os.getenv( + "STAC_FIELD_PROP_DATETIME", "properties.datetime" + ) + PROPERTIES_START_DATETIME_FIELD = os.getenv( + "STAC_FIELD_PROP_START_DATETIME", "properties.start_datetime" + ) + PROPERTIES_END_DATETIME_FIELD = os.getenv( + "STAC_FIELD_PROP_END_DATETIME", "properties.end_datetime" + ) + COLLECTION_FIELD = os.getenv("STAC_FIELD_COLLECTION", "collection") + GEOMETRY_FIELD = os.getenv("STAC_FIELD_GEOMETRY", "geometry") @staticmethod def __nested_field__(field: str):