From 2a8dd32bfadbd2a69ba3bd15002809c69ba8e43b Mon Sep 17 00:00:00 2001 From: Yuri Zmytrakov Date: Tue, 23 Sep 2025 20:42:32 +0200 Subject: [PATCH 1/7] dummy --- Makefile | 8 +- docker-compose.redis.yml | 27 ++++ mypy.ini | 3 + stac_fastapi/core/setup.py | 1 + stac_fastapi/core/stac_fastapi/core/core.py | 67 ++++++++++ .../core/stac_fastapi/core/redis_utils.py | 124 ++++++++++++++++++ 6 files changed, 226 insertions(+), 4 deletions(-) create mode 100644 docker-compose.redis.yml create mode 100644 mypy.ini create mode 100644 stac_fastapi/core/stac_fastapi/core/redis_utils.py diff --git a/Makefile b/Makefile index 204b31a1..317aa630 100644 --- a/Makefile +++ b/Makefile @@ -63,22 +63,22 @@ docker-shell-os: .PHONY: test-elasticsearch test-elasticsearch: - -$(run_es) /bin/bash -c 'export && ./scripts/wait-for-it-es.sh elasticsearch:9200 && cd stac_fastapi/tests/ && pytest' + -$(run_es) /bin/bash -c 'pip install redis==6.4.0 export && ./scripts/wait-for-it-es.sh elasticsearch:9200 && cd stac_fastapi/tests/ && pytest' docker compose down .PHONY: test-opensearch test-opensearch: - -$(run_os) /bin/bash -c 'export && ./scripts/wait-for-it-es.sh opensearch:9202 && cd stac_fastapi/tests/ && pytest' + -$(run_os) /bin/bash -c 'pip install redis==6.4.0 export && ./scripts/wait-for-it-es.sh opensearch:9202 && cd stac_fastapi/tests/ && pytest' docker compose down .PHONY: test-datetime-filtering-es test-datetime-filtering-es: - -$(run_es) /bin/bash -c 'export ENABLE_DATETIME_INDEX_FILTERING=true && ./scripts/wait-for-it-es.sh elasticsearch:9200 && cd stac_fastapi/tests/ && pytest -s --cov=stac_fastapi --cov-report=term-missing -m datetime_filtering' + -$(run_es) /bin/bash -c 'pip install redis==6.4.0 && export ENABLE_DATETIME_INDEX_FILTERING=true && ./scripts/wait-for-it-es.sh elasticsearch:9200 && cd stac_fastapi/tests/ && pytest -s --cov=stac_fastapi --cov-report=term-missing -m datetime_filtering' docker compose down .PHONY: test-datetime-filtering-os test-datetime-filtering-os: - -$(run_os) /bin/bash -c 'export ENABLE_DATETIME_INDEX_FILTERING=true && ./scripts/wait-for-it-es.sh opensearch:9202 && cd stac_fastapi/tests/ && pytest -s --cov=stac_fastapi --cov-report=term-missing -m datetime_filtering' + -$(run_os) /bin/bash -c 'pip install redis==6.4.0 && export ENABLE_DATETIME_INDEX_FILTERING=true && ./scripts/wait-for-it-es.sh opensearch:9202 && cd stac_fastapi/tests/ && pytest -s --cov=stac_fastapi --cov-report=term-missing -m datetime_filtering' docker compose down .PHONY: test diff --git a/docker-compose.redis.yml b/docker-compose.redis.yml new file mode 100644 index 00000000..14686897 --- /dev/null +++ b/docker-compose.redis.yml @@ -0,0 +1,27 @@ +version: '3.8' + +services: + redis: + image: redis:7-alpine + container_name: stac-fastapi-redis + ports: + - "6379:6379" + command: redis-server --appendonly yes + volumes: + - redis_data:/data + environment: + - REDIS_PORT=6379 + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 5s + retries: 3 + networks: + - stac-network + +volumes: + redis_data: + +networks: + stac-network: + driver: bridge \ No newline at end of file diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 00000000..a98660ec --- /dev/null +++ b/mypy.ini @@ -0,0 +1,3 @@ +[mypy] +[mypy-redis.*] +ignore_missing_imports = True \ No newline at end of file diff --git a/stac_fastapi/core/setup.py b/stac_fastapi/core/setup.py index 92442997..b055eecd 100644 --- a/stac_fastapi/core/setup.py +++ b/stac_fastapi/core/setup.py @@ -19,6 +19,7 @@ "pygeofilter~=0.3.1", "jsonschema~=4.0.0", "slowapi~=0.1.9", + "redis==6.4.0", ] setup( diff --git a/stac_fastapi/core/stac_fastapi/core/core.py b/stac_fastapi/core/stac_fastapi/core/core.py index a38bdddb..8484397d 100644 --- a/stac_fastapi/core/stac_fastapi/core/core.py +++ b/stac_fastapi/core/stac_fastapi/core/core.py @@ -24,6 +24,11 @@ from stac_fastapi.core.base_settings import ApiBaseSettings from stac_fastapi.core.datetime_utils import format_datetime_range from stac_fastapi.core.models.links import PagingLinks +from stac_fastapi.core.redis_utils import ( + connect_redis_sentinel, + get_prev_link, + save_self_link, +) from stac_fastapi.core.serializers import CollectionSerializer, ItemSerializer from stac_fastapi.core.session import Session from stac_fastapi.core.utilities import filter_fields @@ -255,6 +260,13 @@ async def all_collections( if parsed_sort: sort = parsed_sort + current_url = str(request.url) + redis = None + try: + redis = await connect_redis_sentinel() + except Exception: + redis = None + collections, next_token = await self.database.get_all_collections( token=token, limit=limit, request=request, sort=sort ) @@ -269,6 +281,22 @@ async def all_collections( }, ] + if redis: + if next_token: + await save_self_link(redis, next_token, current_url) + + prev_link = await get_prev_link(redis, token) + if prev_link: + links.insert( + 0, + { + "rel": "prev", + "type": "application/json", + "method": "GET", + "href": prev_link, + }, + ) + if next_token: next_link = PagingLinks(next=next_token, request=request).link_next() links.append(next_link) @@ -499,6 +527,10 @@ async def post_search( HTTPException: If there is an error with the cql2_json filter. """ base_url = str(request.base_url) + try: + redis = await connect_redis_sentinel() + except Exception: + redis = None search = self.database.make_search() @@ -609,6 +641,41 @@ async def post_search( ] links = await PagingLinks(request=request, next=next_token).get_links() + collection_links = [] + if search_request.collections: + for collection_id in search_request.collections: + collection_links.extend( + [ + { + "rel": "collection", + "type": "application/json", + "href": urljoin(base_url, f"collections/{collection_id}"), + }, + { + "rel": "parent", + "type": "application/json", + "href": urljoin(base_url, f"collections/{collection_id}"), + }, + ] + ) + links.extend(collection_links) + + if redis: + self_link = str(request.url) + await save_self_link(redis, next_token, self_link) + + prev_link = await get_prev_link(redis, token_param) + if prev_link: + links.insert( + 0, + { + "rel": "prev", + "type": "application/json", + "method": "GET", + "href": prev_link, + }, + ) + return stac_types.ItemCollection( type="FeatureCollection", features=items, diff --git a/stac_fastapi/core/stac_fastapi/core/redis_utils.py b/stac_fastapi/core/stac_fastapi/core/redis_utils.py new file mode 100644 index 00000000..e0ab6d11 --- /dev/null +++ b/stac_fastapi/core/stac_fastapi/core/redis_utils.py @@ -0,0 +1,124 @@ +"""Utilities for connecting to and managing Redis connections.""" + +from typing import Optional + +from pydantic_settings import BaseSettings +from redis import asyncio as aioredis +from redis.asyncio.sentinel import Sentinel + +redis_pool: Optional[aioredis.Redis] = None + + +class RedisSentinelSettings(BaseSettings): + """Configuration for connecting to Redis Sentinel.""" + + REDIS_SENTINEL_HOSTS: str = "" + REDIS_SENTINEL_PORTS: str = "26379" + REDIS_SENTINEL_MASTER_NAME: str = "master" + REDIS_DB: int = 15 + + REDIS_MAX_CONNECTIONS: int = 10 + REDIS_RETRY_TIMEOUT: bool = True + REDIS_DECODE_RESPONSES: bool = True + REDIS_CLIENT_NAME: str = "stac-fastapi-app" + REDIS_HEALTH_CHECK_INTERVAL: int = 30 + + +class RedisSettings(BaseSettings): + """Configuration for connecting Redis Sentinel.""" + + REDIS_HOST: str = "" + REDIS_PORT: int = 6379 + REDIS_DB: int = 0 + + REDIS_MAX_CONNECTIONS: int = 10 + REDIS_RETRY_TIMEOUT: bool = True + REDIS_DECODE_RESPONSES: bool = True + REDIS_CLIENT_NAME: str = "stac-fastapi-app" + REDIS_HEALTH_CHECK_INTERVAL: int = 30 + + +# Select the Redis or Redis Sentinel configuration +redis_settings: BaseSettings = RedisSentinelSettings() + + +async def connect_redis(settings: Optional[RedisSettings] = None) -> aioredis.Redis: + """Return a Redis connection.""" + global redis_pool + settings = settings or redis_settings + + if not settings.REDIS_HOST or not settings.REDIS_PORT: + return None + + if redis_pool is None: + pool = aioredis.ConnectionPool( + host=settings.REDIS_HOST, + port=settings.REDIS_PORT, + db=settings.REDIS_DB, + max_connections=settings.REDIS_MAX_CONNECTIONS, + decode_responses=settings.REDIS_DECODE_RESPONSES, + retry_on_timeout=settings.REDIS_RETRY_TIMEOUT, + health_check_interval=settings.REDIS_HEALTH_CHECK_INTERVAL, + ) + redis_pool = aioredis.Redis( + connection_pool=pool, client_name=settings.REDIS_CLIENT_NAME + ) + return redis_pool + + +async def connect_redis_sentinel( + settings: Optional[RedisSentinelSettings] = None, +) -> Optional[aioredis.Redis]: + """Return a Redis Sentinel connection.""" + global redis_pool + + settings = settings or redis_settings + + if ( + not settings.REDIS_SENTINEL_HOSTS + or not settings.REDIS_SENTINEL_PORTS + or not settings.REDIS_SENTINEL_MASTER_NAME + ): + return None + + hosts = [h.strip() for h in settings.REDIS_SENTINEL_HOSTS.split(",") if h.strip()] + ports = [ + int(p.strip()) for p in settings.REDIS_SENTINEL_PORTS.split(",") if p.strip() + ] + + if redis_pool is None: + try: + sentinel = Sentinel( + [(h, p) for h, p in zip(hosts, ports)], + decode_responses=settings.REDIS_DECODE_RESPONSES, + ) + master = sentinel.master_for( + service_name=settings.REDIS_SENTINEL_MASTER_NAME, + db=settings.REDIS_DB, + decode_responses=settings.REDIS_DECODE_RESPONSES, + retry_on_timeout=settings.REDIS_RETRY_TIMEOUT, + client_name=settings.REDIS_CLIENT_NAME, + max_connections=settings.REDIS_MAX_CONNECTIONS, + health_check_interval=settings.REDIS_HEALTH_CHECK_INTERVAL, + ) + redis_pool = master + + except Exception: + return None + + return redis_pool + + +async def save_self_link( + redis: aioredis.Redis, token: Optional[str], self_href: str +) -> None: + """Save the self link for the current token with 30 min TTL.""" + if token: + await redis.setex(f"nav:self:{token}", 1800, self_href) + + +async def get_prev_link(redis: aioredis.Redis, token: Optional[str]) -> Optional[str]: + """Get the previous page link for the current token (if exists).""" + if not token: + return None + return await redis.get(f"nav:self:{token}") From e1d1df1923943d9c4b990241f2ce99befcf64d29 Mon Sep 17 00:00:00 2001 From: Jonathan Healy Date: Wed, 24 Sep 2025 14:09:26 +0800 Subject: [PATCH 2/7] Get collections search fields (#465) **Related Issue(s):** - #461 **Description:** - GET `/collections` collection search fields extension ex. `/collections?fields=id,title`. [#465](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/465) - Improved error messages for sorting on unsortable fields in collection search, including guidance on how to make fields sortable. [#465](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/465) - Added field alias for `temporal` to enable easier sorting by temporal extent, alongside `extent.temporal.interval`. [#465](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/465) **PR Checklist:** - [x] Code is formatted and linted (run `pre-commit run --all-files`) - [x] Tests pass (run `make test`) - [x] Documentation has been updated to reflect changes, if applicable - [x] Changes are added to the changelog --- CHANGELOG.md | 3 + README.md | 28 ++++++- stac_fastapi/core/stac_fastapi/core/core.py | 24 +++++- .../stac_fastapi/elasticsearch/app.py | 2 +- .../elasticsearch/database_logic.py | 15 ++++ .../opensearch/stac_fastapi/opensearch/app.py | 2 +- .../stac_fastapi/opensearch/database_logic.py | 15 ++++ .../stac_fastapi/sfeos_helpers/mappings.py | 2 + .../tests/api/test_api_search_collections.py | 75 +++++++++++++++++++ 9 files changed, 160 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e79dca0e..b397fc3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Added `USE_DATETIME` environment variable to configure datetime search behavior in SFEOS. [#452](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/452) - GET `/collections` collection search sort extension ex. `/collections?sortby=+id`. [#456](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/456) +- GET `/collections` collection search fields extension ex. `/collections?fields=id,title`. [#465](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/465) +- Improved error messages for sorting on unsortable fields in collection search, including guidance on how to make fields sortable. [#465](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/465) +- Added field alias for `temporal` to enable easier sorting by temporal extent, alongside `extent.temporal.interval`. [#465](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/465) ### Changed diff --git a/README.md b/README.md index 578a440a..929d17c6 100644 --- a/README.md +++ b/README.md @@ -36,11 +36,10 @@ SFEOS (stac-fastapi-elasticsearch-opensearch) is a high-performance, scalable AP - **Scale to millions of geospatial assets** with fast search performance through optimized spatial indexing and query capabilities - **Support OGC-compliant filtering** including spatial operations (intersects, contains, etc.) and temporal queries - **Perform geospatial aggregations** to analyze data distribution across space and time +- **Enhanced collection search capabilities** with support for sorting and field selection This implementation builds on the STAC-FastAPI framework, providing a production-ready solution specifically optimized for Elasticsearch and OpenSearch databases. It's ideal for organizations managing large geospatial data catalogs who need efficient discovery and access capabilities through standardized APIs. - - ## Common Deployment Patterns stac-fastapi-elasticsearch-opensearch can be deployed in several ways depending on your needs: @@ -72,6 +71,7 @@ This project is built on the following technologies: STAC, stac-fastapi, FastAPI - [Common Deployment Patterns](#common-deployment-patterns) - [Technologies](#technologies) - [Table of Contents](#table-of-contents) + - [Collection Search Extensions](#collection-search-extensions) - [Documentation \& Resources](#documentation--resources) - [Package Structure](#package-structure) - [Examples](#examples) @@ -113,6 +113,30 @@ This project is built on the following technologies: STAC, stac-fastapi, FastAPI - [Gitter Chat](https://app.gitter.im/#/room/#stac-fastapi-elasticsearch_community:gitter.im) - For real-time discussions - [GitHub Discussions](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/discussions) - For longer-form questions and answers +## Collection Search Extensions + +SFEOS implements extended capabilities for the `/collections` endpoint, allowing for more powerful collection discovery: + +- **Sorting**: Sort collections by sortable fields using the `sortby` parameter + - Example: `/collections?sortby=+id` (ascending sort by ID) + - Example: `/collections?sortby=-id` (descending sort by ID) + - Example: `/collections?sortby=-temporal` (descending sort by temporal extent) + +- **Field Selection**: Request only specific fields to be returned using the `fields` parameter + - Example: `/collections?fields=id,title,description` + - This helps reduce payload size when only certain fields are needed + +These extensions make it easier to build user interfaces that display and navigate through collections efficiently. + +> **Note**: Sorting is only available on fields that are indexed for sorting in Elasticsearch/OpenSearch. With the default mappings, you can sort on: +> - `id` (keyword field) +> - `extent.temporal.interval` (date field) +> - `temporal` (alias to extent.temporal.interval) +> +> Text fields like `title` and `description` are not sortable by default as they use text analysis for better search capabilities. Attempting to sort on these fields will result in a user-friendly error message explaining which fields are sortable and how to make additional fields sortable by updating the mappings. +> +> **Important**: Adding keyword fields to make text fields sortable can significantly increase the index size, especially for large text fields. Consider the storage implications when deciding which fields to make sortable. + ## Package Structure This project is organized into several packages, each with a specific purpose: diff --git a/stac_fastapi/core/stac_fastapi/core/core.py b/stac_fastapi/core/stac_fastapi/core/core.py index 8484397d..4590644a 100644 --- a/stac_fastapi/core/stac_fastapi/core/core.py +++ b/stac_fastapi/core/stac_fastapi/core/core.py @@ -230,11 +230,13 @@ async def landing_page(self, **kwargs) -> stac_types.LandingPage: return landing_page async def all_collections( - self, sortby: Optional[str] = None, **kwargs + self, fields: Optional[List[str]] = None, sortby: Optional[str] = None, **kwargs ) -> stac_types.Collections: """Read all collections from the database. Args: + fields (Optional[List[str]]): Fields to include or exclude from the results. + sortby (Optional[str]): Sorting options for the results. **kwargs: Keyword arguments from the request. Returns: @@ -245,6 +247,15 @@ async def all_collections( limit = int(request.query_params.get("limit", os.getenv("STAC_ITEM_LIMIT", 10))) token = request.query_params.get("token") + # Process fields parameter for filtering collection properties + includes, excludes = set(), set() + if fields and self.extension_is_enabled("FieldsExtension"): + for field in fields: + if field[0] == "-": + excludes.add(field[1:]) + else: + includes.add(field[1:] if field[0] in "+ " else field) + sort = None if sortby: parsed_sort = [] @@ -271,6 +282,15 @@ async def all_collections( token=token, limit=limit, request=request, sort=sort ) + # Apply field filtering if fields parameter was provided + if fields and self.extension_is_enabled("FieldsExtension"): + filtered_collections = [ + filter_fields(collection, includes, excludes) + for collection in collections + ] + else: + filtered_collections = collections + links = [ {"rel": Relations.root.value, "type": MimeTypes.json, "href": base_url}, {"rel": Relations.parent.value, "type": MimeTypes.json, "href": base_url}, @@ -301,7 +321,7 @@ async def all_collections( next_link = PagingLinks(next=next_token, request=request).link_next() links.append(next_link) - return stac_types.Collections(collections=collections, links=links) + return stac_types.Collections(collections=filtered_collections, links=links) async def get_collection( self, collection_id: str, **kwargs diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py index ea26c022..67600072 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py @@ -120,7 +120,7 @@ collection_search_extensions = [ # QueryExtension(conformance_classes=[QueryConformanceClasses.COLLECTIONS]), SortExtension(conformance_classes=[SortConformanceClasses.COLLECTIONS]), - # FieldsExtension(conformance_classes=[FieldsConformanceClasses.COLLECTIONS]), + FieldsExtension(conformance_classes=[FieldsConformanceClasses.COLLECTIONS]), # CollectionSearchFilterExtension( # conformance_classes=[FilterConformanceClasses.COLLECTIONS] # ), diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index df1e816d..35cd8d9e 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -186,13 +186,28 @@ async def get_all_collections( Returns: A tuple of (collections, next pagination token if any). + + Raises: + HTTPException: If sorting is requested on a field that is not sortable. """ + # Define sortable fields based on the ES_COLLECTIONS_MAPPINGS + sortable_fields = ["id", "extent.temporal.interval", "temporal"] + + # Format the sort parameter formatted_sort = [] if sort: for item in sort: field = item.get("field") direction = item.get("direction", "asc") if field: + # Validate that the field is sortable + if field not in sortable_fields: + raise HTTPException( + status_code=400, + detail=f"Field '{field}' is not sortable. Sortable fields are: {', '.join(sortable_fields)}. " + + "Text fields are not sortable by default in Elasticsearch. " + + "To make a field sortable, update the mapping to use 'keyword' type or add a '.keyword' subfield. ", + ) formatted_sort.append({field: {"order": direction}}) # Always include id as a secondary sort to ensure consistent pagination if not any("id" in item for item in formatted_sort): diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py index 789cb728..7d9f5d91 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py @@ -120,7 +120,7 @@ collection_search_extensions = [ # QueryExtension(conformance_classes=[QueryConformanceClasses.COLLECTIONS]), SortExtension(conformance_classes=[SortConformanceClasses.COLLECTIONS]), - # FieldsExtension(conformance_classes=[FieldsConformanceClasses.COLLECTIONS]), + FieldsExtension(conformance_classes=[FieldsConformanceClasses.COLLECTIONS]), # CollectionSearchFilterExtension( # conformance_classes=[FilterConformanceClasses.COLLECTIONS] # ), diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py index 4253a00a..94a95b32 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py @@ -170,13 +170,28 @@ async def get_all_collections( Returns: A tuple of (collections, next pagination token if any). + + Raises: + HTTPException: If sorting is requested on a field that is not sortable. """ + # Define sortable fields based on the ES_COLLECTIONS_MAPPINGS + sortable_fields = ["id", "extent.temporal.interval", "temporal"] + + # Format the sort parameter formatted_sort = [] if sort: for item in sort: field = item.get("field") direction = item.get("direction", "asc") if field: + # Validate that the field is sortable + if field not in sortable_fields: + raise HTTPException( + status_code=400, + detail=f"Field '{field}' is not sortable. Sortable fields are: {', '.join(sortable_fields)}. " + + "Text fields are not sortable by default in OpenSearch. " + + "To make a field sortable, update the mapping to use 'keyword' type or add a '.keyword' subfield. ", + ) formatted_sort.append({field: {"order": direction}}) # Always include id as a secondary sort to ensure consistent pagination if not any("id" in item for item in formatted_sort): diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/mappings.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/mappings.py index 17cdd1ea..df002dc5 100644 --- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/mappings.py +++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/mappings.py @@ -165,6 +165,8 @@ class Geometry(Protocol): # noqa "providers": {"type": "object", "enabled": False}, "links": {"type": "object", "enabled": False}, "item_assets": {"type": "object", "enabled": get_bool_env("STAC_INDEX_ASSETS")}, + # Field alias to allow sorting on 'temporal' (points to extent.temporal.interval) + "temporal": {"type": "alias", "path": "extent.temporal.interval"}, }, } diff --git a/stac_fastapi/tests/api/test_api_search_collections.py b/stac_fastapi/tests/api/test_api_search_collections.py index ed0dfc1b..ffd84831 100644 --- a/stac_fastapi/tests/api/test_api_search_collections.py +++ b/stac_fastapi/tests/api/test_api_search_collections.py @@ -77,3 +77,78 @@ async def test_collections_sort_id_desc(app_client, txn_client, load_test_data): assert len(test_collections) == len(collection_ids) for i, expected_id in enumerate(sorted_ids): assert test_collections[i]["id"] == expected_id + + +@pytest.mark.asyncio +async def test_collections_fields(app_client, txn_client, load_test_data): + """Verify GET /collections honors the fields parameter.""" + # Create multiple collections with different ids + base_collection = load_test_data("test_collection.json") + + # Create collections with ids in a specific order to test fields + # Use unique prefixes to avoid conflicts between tests + test_prefix = f"fields-{uuid.uuid4().hex[:8]}" + collection_ids = [f"{test_prefix}-a", f"{test_prefix}-b", f"{test_prefix}-c"] + + for i, coll_id in enumerate(collection_ids): + test_collection = base_collection.copy() + test_collection["id"] = coll_id + test_collection["title"] = f"Test Collection {i}" + test_collection["description"] = f"Description for collection {i}" + await create_collection(txn_client, test_collection) + + # Test include fields parameter + resp = await app_client.get( + "/collections", + params=[("fields", "id"), ("fields", "title")], + ) + assert resp.status_code == 200 + resp_json = resp.json() + + # Check if collections exist in the response + assert "collections" in resp_json, "No collections in response" + + # Filter collections to only include the ones we created for this test + test_collections = [] + for c in resp_json["collections"]: + if "id" in c and c["id"].startswith(test_prefix): + test_collections.append(c) + + # Filter collections to only include the ones we created for this test + test_collections = [] + for c in resp_json["collections"]: + if "id" in c and c["id"].startswith(test_prefix): + test_collections.append(c) + + # Collections should only have id and title fields + for collection in test_collections: + assert "id" in collection + assert "title" in collection + assert "description" not in collection + assert "links" in collection # links are always included + + # Test exclude fields parameter + resp = await app_client.get( + "/collections", + params=[("fields", "-description")], + ) + assert resp.status_code == 200 + resp_json = resp.json() + + # Check if collections exist in the response + assert ( + "collections" in resp_json + ), "No collections in response for exclude fields test" + + # Filter collections to only include the ones we created for this test + test_collections = [] + for c in resp_json["collections"]: + if "id" in c and c["id"].startswith(test_prefix): + test_collections.append(c) + + # Collections should have all fields except description + for collection in test_collections: + assert "id" in collection + assert "title" in collection + assert "description" not in collection + assert "links" in collection From ce672b052e5771c5e16fd57635b0b0499b7cfa23 Mon Sep 17 00:00:00 2001 From: Jonathan Healy Date: Thu, 25 Sep 2025 00:16:32 +0800 Subject: [PATCH 3/7] Enable collections search ENV variable (#469) **Related Issue(s):** - #468 **Description:** **PR Checklist:** - [x] Code is formatted and linted (run `pre-commit run --all-files`) - [x] Tests pass (run `make test`) - [x] Documentation has been updated to reflect changes, if applicable - [x] Changes are added to the changelog --- CHANGELOG.md | 1 + README.md | 7 + .../stac_fastapi/elasticsearch/app.py | 44 +++-- .../opensearch/stac_fastapi/opensearch/app.py | 44 +++-- .../tests/api/test_collections_search_env.py | 167 ++++++++++++++++++ 5 files changed, 225 insertions(+), 38 deletions(-) create mode 100644 stac_fastapi/tests/api/test_collections_search_env.py diff --git a/CHANGELOG.md b/CHANGELOG.md index b397fc3e..8396708a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - GET `/collections` collection search fields extension ex. `/collections?fields=id,title`. [#465](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/465) - Improved error messages for sorting on unsortable fields in collection search, including guidance on how to make fields sortable. [#465](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/465) - Added field alias for `temporal` to enable easier sorting by temporal extent, alongside `extent.temporal.interval`. [#465](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/465) +- Added `ENABLE_COLLECTIONS_SEARCH` environment variable to make collection search extensions optional (defaults to enabled). [#465](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/465) ### Changed diff --git a/README.md b/README.md index 929d17c6..c1be16ae 100644 --- a/README.md +++ b/README.md @@ -128,6 +128,8 @@ SFEOS implements extended capabilities for the `/collections` endpoint, allowing These extensions make it easier to build user interfaces that display and navigate through collections efficiently. +> **Configuration**: Collection search extensions can be disabled by setting the `ENABLE_COLLECTIONS_SEARCH` environment variable to `false`. By default, these extensions are enabled. + > **Note**: Sorting is only available on fields that are indexed for sorting in Elasticsearch/OpenSearch. With the default mappings, you can sort on: > - `id` (keyword field) > - `extent.temporal.interval` (date field) @@ -267,6 +269,7 @@ You can customize additional settings in your `.env` file: | `ENABLE_DIRECT_RESPONSE` | Enable direct response for maximum performance (disables all FastAPI dependencies, including authentication, custom status codes, and validation) | `false` | Optional | | `RAISE_ON_BULK_ERROR` | Controls whether bulk insert operations raise exceptions on errors. If set to `true`, the operation will stop and raise an exception when an error occurs. If set to `false`, errors will be logged, and the operation will continue. **Note:** STAC Item and ItemCollection validation errors will always raise, regardless of this flag. | `false` | Optional | | `DATABASE_REFRESH` | Controls whether database operations refresh the index immediately after changes. If set to `true`, changes will be immediately searchable. If set to `false`, changes may not be immediately visible but can improve performance for bulk operations. If set to `wait_for`, changes will wait for the next refresh cycle to become visible. | `false` | Optional | +| `ENABLE_COLLECTIONS_SEARCH` | Enable collection search extensions (sort, fields). | `true` | Optional | | `ENABLE_TRANSACTIONS_EXTENSIONS` | Enables or disables the Transactions and Bulk Transactions API extensions. If set to `false`, the POST `/collections` route and related transaction endpoints (including bulk transaction operations) will be unavailable in the API. This is useful for deployments where mutating the catalog via the API should be prevented. | `true` | Optional | | `STAC_ITEM_LIMIT` | Sets the environment variable for result limiting to SFEOS for the number of returned items and STAC collections. | `10` | Optional | | `STAC_INDEX_ASSETS` | Controls if Assets are indexed when added to Elasticsearch/Opensearch. This allows asset fields to be included in search queries. | `false` | Optional | @@ -413,6 +416,10 @@ The system uses a precise naming convention: - **Root Path Configuration**: The application root path is the base URL by default. - For AWS Lambda with Gateway API: Set `STAC_FASTAPI_ROOT_PATH` to match the Gateway API stage name (e.g., `/v1`) +- **Feature Configuration**: Control which features are enabled: + - `ENABLE_COLLECTIONS_SEARCH`: Set to `true` (default) to enable collection search extensions (sort, fields). Set to `false` to disable. + - `ENABLE_TRANSACTIONS_EXTENSIONS`: Set to `true` (default) to enable transaction extensions. Set to `false` to disable. + ## Collection Pagination diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py index 67600072..25c865f8 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py @@ -57,7 +57,9 @@ logger = logging.getLogger(__name__) TRANSACTIONS_EXTENSIONS = get_bool_env("ENABLE_TRANSACTIONS_EXTENSIONS", default=True) +ENABLE_COLLECTIONS_SEARCH = get_bool_env("ENABLE_COLLECTIONS_SEARCH", default=True) logger.info("TRANSACTIONS_EXTENSIONS is set to %s", TRANSACTIONS_EXTENSIONS) +logger.info("ENABLE_COLLECTIONS_SEARCH is set to %s", ENABLE_COLLECTIONS_SEARCH) settings = ElasticsearchSettings() session = Session.create_from_settings(settings) @@ -115,25 +117,26 @@ extensions = [aggregation_extension] + search_extensions -# Create collection search extensions -# Only sort extension is enabled for now -collection_search_extensions = [ - # QueryExtension(conformance_classes=[QueryConformanceClasses.COLLECTIONS]), - SortExtension(conformance_classes=[SortConformanceClasses.COLLECTIONS]), - FieldsExtension(conformance_classes=[FieldsConformanceClasses.COLLECTIONS]), - # CollectionSearchFilterExtension( - # conformance_classes=[FilterConformanceClasses.COLLECTIONS] - # ), - # FreeTextExtension(conformance_classes=[FreeTextConformanceClasses.COLLECTIONS]), -] - -# Initialize collection search with its extensions -collection_search_ext = CollectionSearchExtension.from_extensions( - collection_search_extensions -) -collections_get_request_model = collection_search_ext.GET +# Create collection search extensions if enabled +if ENABLE_COLLECTIONS_SEARCH: + # Create collection search extensions + collection_search_extensions = [ + # QueryExtension(conformance_classes=[QueryConformanceClasses.COLLECTIONS]), + SortExtension(conformance_classes=[SortConformanceClasses.COLLECTIONS]), + FieldsExtension(conformance_classes=[FieldsConformanceClasses.COLLECTIONS]), + # CollectionSearchFilterExtension( + # conformance_classes=[FilterConformanceClasses.COLLECTIONS] + # ), + # FreeTextExtension(conformance_classes=[FreeTextConformanceClasses.COLLECTIONS]), + ] + + # Initialize collection search with its extensions + collection_search_ext = CollectionSearchExtension.from_extensions( + collection_search_extensions + ) + collections_get_request_model = collection_search_ext.GET -extensions.append(collection_search_ext) + extensions.append(collection_search_ext) database_logic.extensions = [type(ext).__name__ for ext in extensions] @@ -170,10 +173,13 @@ "search_get_request_model": create_get_request_model(search_extensions), "search_post_request_model": post_request_model, "items_get_request_model": items_get_request_model, - "collections_get_request_model": collections_get_request_model, "route_dependencies": get_route_dependencies(), } +# Add collections_get_request_model if collection search is enabled +if ENABLE_COLLECTIONS_SEARCH: + app_config["collections_get_request_model"] = collections_get_request_model + api = StacApi(**app_config) diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py index 7d9f5d91..362eb9bf 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py @@ -57,7 +57,9 @@ logger = logging.getLogger(__name__) TRANSACTIONS_EXTENSIONS = get_bool_env("ENABLE_TRANSACTIONS_EXTENSIONS", default=True) +ENABLE_COLLECTIONS_SEARCH = get_bool_env("ENABLE_COLLECTIONS_SEARCH", default=True) logger.info("TRANSACTIONS_EXTENSIONS is set to %s", TRANSACTIONS_EXTENSIONS) +logger.info("ENABLE_COLLECTIONS_SEARCH is set to %s", ENABLE_COLLECTIONS_SEARCH) settings = OpensearchSettings() session = Session.create_from_settings(settings) @@ -115,25 +117,26 @@ extensions = [aggregation_extension] + search_extensions -# Create collection search extensions -# Only sort extension is enabled for now -collection_search_extensions = [ - # QueryExtension(conformance_classes=[QueryConformanceClasses.COLLECTIONS]), - SortExtension(conformance_classes=[SortConformanceClasses.COLLECTIONS]), - FieldsExtension(conformance_classes=[FieldsConformanceClasses.COLLECTIONS]), - # CollectionSearchFilterExtension( - # conformance_classes=[FilterConformanceClasses.COLLECTIONS] - # ), - # FreeTextExtension(conformance_classes=[FreeTextConformanceClasses.COLLECTIONS]), -] - -# Initialize collection search with its extensions -collection_search_ext = CollectionSearchExtension.from_extensions( - collection_search_extensions -) -collections_get_request_model = collection_search_ext.GET +# Create collection search extensions if enabled +if ENABLE_COLLECTIONS_SEARCH: + # Create collection search extensions + collection_search_extensions = [ + # QueryExtension(conformance_classes=[QueryConformanceClasses.COLLECTIONS]), + SortExtension(conformance_classes=[SortConformanceClasses.COLLECTIONS]), + FieldsExtension(conformance_classes=[FieldsConformanceClasses.COLLECTIONS]), + # CollectionSearchFilterExtension( + # conformance_classes=[FilterConformanceClasses.COLLECTIONS] + # ), + # FreeTextExtension(conformance_classes=[FreeTextConformanceClasses.COLLECTIONS]), + ] + + # Initialize collection search with its extensions + collection_search_ext = CollectionSearchExtension.from_extensions( + collection_search_extensions + ) + collections_get_request_model = collection_search_ext.GET -extensions.append(collection_search_ext) + extensions.append(collection_search_ext) database_logic.extensions = [type(ext).__name__ for ext in extensions] @@ -167,13 +170,16 @@ post_request_model=post_request_model, landing_page_id=os.getenv("STAC_FASTAPI_LANDING_PAGE_ID", "stac-fastapi"), ), - "collections_get_request_model": collections_get_request_model, "search_get_request_model": create_get_request_model(search_extensions), "search_post_request_model": post_request_model, "items_get_request_model": items_get_request_model, "route_dependencies": get_route_dependencies(), } +# Add collections_get_request_model if collection search is enabled +if ENABLE_COLLECTIONS_SEARCH: + app_config["collections_get_request_model"] = collections_get_request_model + api = StacApi(**app_config) diff --git a/stac_fastapi/tests/api/test_collections_search_env.py b/stac_fastapi/tests/api/test_collections_search_env.py new file mode 100644 index 00000000..5358faf9 --- /dev/null +++ b/stac_fastapi/tests/api/test_collections_search_env.py @@ -0,0 +1,167 @@ +"""Test the ENABLE_COLLECTIONS_SEARCH environment variable.""" + +import os +import uuid +from unittest import mock + +import pytest + +from ..conftest import create_collection, refresh_indices + + +@pytest.mark.asyncio +@mock.patch.dict(os.environ, {"ENABLE_COLLECTIONS_SEARCH": "false"}) +async def test_collections_search_disabled(app_client, txn_client, load_test_data): + """Test that collection search extensions are disabled when ENABLE_COLLECTIONS_SEARCH=false.""" + # Create multiple collections with different ids to test sorting + base_collection = load_test_data("test_collection.json") + + # Use unique prefixes to avoid conflicts between tests + test_prefix = f"disabled-{uuid.uuid4().hex[:8]}" + collection_ids = [f"{test_prefix}-c", f"{test_prefix}-a", f"{test_prefix}-b"] + + for i, coll_id in enumerate(collection_ids): + test_collection = base_collection.copy() + test_collection["id"] = coll_id + test_collection["title"] = f"Test Collection {i}" + await create_collection(txn_client, test_collection) + + # Refresh indices to ensure collections are searchable + await refresh_indices(txn_client) + + # When collection search is disabled, sortby parameter should be ignored + resp = await app_client.get( + "/collections", + params=[("sortby", "+id")], + ) + assert resp.status_code == 200 + + # Verify that results are NOT sorted by id (should be in insertion order or default order) + resp_json = resp.json() + collections = [ + c for c in resp_json["collections"] if c["id"].startswith(test_prefix) + ] + + # Extract the ids in the order they were returned + returned_ids = [c["id"] for c in collections] + + # If sorting was working, they would be in alphabetical order: a, b, c + # But since sorting is disabled, they should be in a different order + # We can't guarantee the exact order, but we can check they're not in alphabetical order + sorted_ids = sorted(returned_ids) + assert ( + returned_ids != sorted_ids or len(collections) < 2 + ), "Collections appear to be sorted despite ENABLE_COLLECTIONS_SEARCH=false" + + # Fields parameter should also be ignored + resp = await app_client.get( + "/collections", + params=[("fields", "id")], # Request only id field + ) + assert resp.status_code == 200 + + # Verify that all fields are still returned, not just id + resp_json = resp.json() + for collection in resp_json["collections"]: + if collection["id"].startswith(test_prefix): + # If fields filtering was working, only id would be present + # Since it's disabled, other fields like title should still be present + assert ( + "title" in collection + ), "Fields filtering appears to be working despite ENABLE_COLLECTIONS_SEARCH=false" + + +@pytest.mark.asyncio +@mock.patch.dict(os.environ, {"ENABLE_COLLECTIONS_SEARCH": "true"}) +async def test_collections_search_enabled(app_client, txn_client, load_test_data): + """Test that collection search extensions work when ENABLE_COLLECTIONS_SEARCH=true.""" + # Create multiple collections with different ids to test sorting + base_collection = load_test_data("test_collection.json") + + # Use unique prefixes to avoid conflicts between tests + test_prefix = f"enabled-{uuid.uuid4().hex[:8]}" + collection_ids = [f"{test_prefix}-c", f"{test_prefix}-a", f"{test_prefix}-b"] + + for i, coll_id in enumerate(collection_ids): + test_collection = base_collection.copy() + test_collection["id"] = coll_id + test_collection["title"] = f"Test Collection {i}" + await create_collection(txn_client, test_collection) + + # Refresh indices to ensure collections are searchable + await refresh_indices(txn_client) + + # Test that sortby parameter works - sort by id ascending + resp = await app_client.get( + "/collections", + params=[("sortby", "+id")], + ) + assert resp.status_code == 200 + + # Verify that results are sorted by id in ascending order + resp_json = resp.json() + collections = [ + c for c in resp_json["collections"] if c["id"].startswith(test_prefix) + ] + + # Extract the ids in the order they were returned + returned_ids = [c["id"] for c in collections] + + # Verify they're in ascending order + assert returned_ids == sorted( + returned_ids + ), "Collections are not sorted by id ascending" + + # Test that sortby parameter works - sort by id descending + resp = await app_client.get( + "/collections", + params=[("sortby", "-id")], + ) + assert resp.status_code == 200 + + # Verify that results are sorted by id in descending order + resp_json = resp.json() + collections = [ + c for c in resp_json["collections"] if c["id"].startswith(test_prefix) + ] + + # Extract the ids in the order they were returned + returned_ids = [c["id"] for c in collections] + + # Verify they're in descending order + assert returned_ids == sorted( + returned_ids, reverse=True + ), "Collections are not sorted by id descending" + + # Test that fields parameter works - request only id field + resp = await app_client.get( + "/collections", + params=[("fields", "id")], + ) + assert resp.status_code == 200 + resp_json = resp.json() + + # When fields=id is specified, collections should only have id field + for collection in resp_json["collections"]: + if collection["id"].startswith(test_prefix): + assert "id" in collection, "id field is missing" + assert ( + "title" not in collection + ), "title field should be excluded when fields=id" + + # Test that fields parameter works - request multiple fields + resp = await app_client.get( + "/collections", + params=[("fields", "id,title")], + ) + assert resp.status_code == 200 + resp_json = resp.json() + + # When fields=id,title is specified, collections should have both fields but not others + for collection in resp_json["collections"]: + if collection["id"].startswith(test_prefix): + assert "id" in collection, "id field is missing" + assert "title" in collection, "title field is missing" + assert ( + "description" not in collection + ), "description field should be excluded when fields=id,title" From 84c10893ba925a5e149a8a19e6be506092a8cc80 Mon Sep 17 00:00:00 2001 From: Jonathan Healy Date: Thu, 25 Sep 2025 00:51:57 +0800 Subject: [PATCH 4/7] GET /collections search free text extension (#470) **Related Issue(s):** - #460 **Description:** ex. `/collections?q=Sentinel-2a` **PR Checklist:** - [x] Code is formatted and linted (run `pre-commit run --all-files`) - [x] Tests pass (run `make test`) - [x] Documentation has been updated to reflect changes, if applicable - [x] Changes are added to the changelog --- CHANGELOG.md | 1 + README.md | 5 ++ stac_fastapi/core/stac_fastapi/core/core.py | 14 +++- .../stac_fastapi/elasticsearch/app.py | 5 +- .../elasticsearch/database_logic.py | 34 +++++++++ .../opensearch/stac_fastapi/opensearch/app.py | 5 +- .../stac_fastapi/opensearch/database_logic.py | 33 +++++++++ .../tests/api/test_api_search_collections.py | 74 +++++++++++++++++++ 8 files changed, 163 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8396708a..d0794819 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Added +- GET `/collections` collection search free text extension ex. `/collections?q=sentinel`. [#470](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/470) - Added `USE_DATETIME` environment variable to configure datetime search behavior in SFEOS. [#452](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/452) - GET `/collections` collection search sort extension ex. `/collections?sortby=+id`. [#456](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/456) - GET `/collections` collection search fields extension ex. `/collections?fields=id,title`. [#465](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/465) diff --git a/README.md b/README.md index c1be16ae..78c02408 100644 --- a/README.md +++ b/README.md @@ -126,6 +126,11 @@ SFEOS implements extended capabilities for the `/collections` endpoint, allowing - Example: `/collections?fields=id,title,description` - This helps reduce payload size when only certain fields are needed +- **Free Text Search**: Search across collection text fields using the `q` parameter + - Example: `/collections?q=landsat` + - Searches across multiple text fields including title, description, and keywords + - Supports partial word matching and relevance-based sorting + These extensions make it easier to build user interfaces that display and navigate through collections efficiently. > **Configuration**: Collection search extensions can be disabled by setting the `ENABLE_COLLECTIONS_SEARCH` environment variable to `false`. By default, these extensions are enabled. diff --git a/stac_fastapi/core/stac_fastapi/core/core.py b/stac_fastapi/core/stac_fastapi/core/core.py index 4590644a..b6f89e64 100644 --- a/stac_fastapi/core/stac_fastapi/core/core.py +++ b/stac_fastapi/core/stac_fastapi/core/core.py @@ -230,13 +230,18 @@ async def landing_page(self, **kwargs) -> stac_types.LandingPage: return landing_page async def all_collections( - self, fields: Optional[List[str]] = None, sortby: Optional[str] = None, **kwargs + self, + fields: Optional[List[str]] = None, + sortby: Optional[str] = None, + q: Optional[Union[str, List[str]]] = None, + **kwargs, ) -> stac_types.Collections: """Read all collections from the database. Args: fields (Optional[List[str]]): Fields to include or exclude from the results. sortby (Optional[str]): Sorting options for the results. + q (Optional[List[str]]): Free text search terms. **kwargs: Keyword arguments from the request. Returns: @@ -278,8 +283,13 @@ async def all_collections( except Exception: redis = None + # Convert q to a list if it's a string + q_list = None + if q is not None: + q_list = [q] if isinstance(q, str) else q + collections, next_token = await self.database.get_all_collections( - token=token, limit=limit, request=request, sort=sort + token=token, limit=limit, request=request, sort=sort, q=q_list ) # Apply field filtering if fields parameter was provided diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py index 25c865f8..18b52147 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py @@ -45,8 +45,7 @@ ) from stac_fastapi.extensions.core.fields import FieldsConformanceClasses from stac_fastapi.extensions.core.filter import FilterConformanceClasses - -# from stac_fastapi.extensions.core.free_text import FreeTextConformanceClasses +from stac_fastapi.extensions.core.free_text import FreeTextConformanceClasses from stac_fastapi.extensions.core.query import QueryConformanceClasses from stac_fastapi.extensions.core.sort import SortConformanceClasses from stac_fastapi.extensions.third_party import BulkTransactionExtension @@ -127,7 +126,7 @@ # CollectionSearchFilterExtension( # conformance_classes=[FilterConformanceClasses.COLLECTIONS] # ), - # FreeTextExtension(conformance_classes=[FreeTextConformanceClasses.COLLECTIONS]), + FreeTextExtension(conformance_classes=[FreeTextConformanceClasses.COLLECTIONS]), ] # Initialize collection search with its extensions diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index 35cd8d9e..c472039b 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -175,6 +175,7 @@ async def get_all_collections( limit: int, request: Request, sort: Optional[List[Dict[str, Any]]] = None, + q: Optional[List[str]] = None, ) -> Tuple[List[Dict[str, Any]], Optional[str]]: """Retrieve a list of collections from Elasticsearch, supporting pagination. @@ -183,6 +184,7 @@ async def get_all_collections( limit (int): The number of results to return. request (Request): The FastAPI request object. sort (Optional[List[Dict[str, Any]]]): Optional sort parameter from the request. + q (Optional[List[str]]): Free text search terms. Returns: A tuple of (collections, next pagination token if any). @@ -223,6 +225,38 @@ async def get_all_collections( if token: body["search_after"] = [token] + # Apply free text query if provided + if q: + # For collections, we want to search across all relevant fields + should_clauses = [] + + # For each search term + for term in q: + # Create a multi_match query for each term + for field in [ + "id", + "title", + "description", + "keywords", + "summaries.platform", + "summaries.constellation", + "providers.name", + "providers.url", + ]: + should_clauses.append( + { + "wildcard": { + field: {"value": f"*{term}*", "case_insensitive": True} + } + } + ) + + # Add the query to the body using bool query with should clauses + body["query"] = { + "bool": {"should": should_clauses, "minimum_should_match": 1} + } + + # Execute the search response = await self.client.search( index=COLLECTIONS_INDEX, body=body, diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py index 362eb9bf..34d55589 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py @@ -39,8 +39,7 @@ ) from stac_fastapi.extensions.core.fields import FieldsConformanceClasses from stac_fastapi.extensions.core.filter import FilterConformanceClasses - -# from stac_fastapi.extensions.core.free_text import FreeTextConformanceClasses +from stac_fastapi.extensions.core.free_text import FreeTextConformanceClasses from stac_fastapi.extensions.core.query import QueryConformanceClasses from stac_fastapi.extensions.core.sort import SortConformanceClasses from stac_fastapi.extensions.third_party import BulkTransactionExtension @@ -127,7 +126,7 @@ # CollectionSearchFilterExtension( # conformance_classes=[FilterConformanceClasses.COLLECTIONS] # ), - # FreeTextExtension(conformance_classes=[FreeTextConformanceClasses.COLLECTIONS]), + FreeTextExtension(conformance_classes=[FreeTextConformanceClasses.COLLECTIONS]), ] # Initialize collection search with its extensions diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py index 94a95b32..f4b8abd0 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py @@ -159,6 +159,7 @@ async def get_all_collections( limit: int, request: Request, sort: Optional[List[Dict[str, Any]]] = None, + q: Optional[List[str]] = None, ) -> Tuple[List[Dict[str, Any]], Optional[str]]: """Retrieve a list of collections from Elasticsearch, supporting pagination. @@ -167,6 +168,7 @@ async def get_all_collections( limit (int): The number of results to return. request (Request): The FastAPI request object. sort (Optional[List[Dict[str, Any]]]): Optional sort parameter from the request. + q (Optional[List[str]]): Free text search terms. Returns: A tuple of (collections, next pagination token if any). @@ -207,6 +209,37 @@ async def get_all_collections( if token: body["search_after"] = [token] + # Apply free text query if provided + if q: + # For collections, we want to search across all relevant fields + should_clauses = [] + + # For each search term + for term in q: + # Create a multi_match query for each term + for field in [ + "id", + "title", + "description", + "keywords", + "summaries.platform", + "summaries.constellation", + "providers.name", + "providers.url", + ]: + should_clauses.append( + { + "wildcard": { + field: {"value": f"*{term}*", "case_insensitive": True} + } + } + ) + + # Add the query to the body using bool query with should clauses + body["query"] = { + "bool": {"should": should_clauses, "minimum_should_match": 1} + } + response = await self.client.search( index=COLLECTIONS_INDEX, body=body, diff --git a/stac_fastapi/tests/api/test_api_search_collections.py b/stac_fastapi/tests/api/test_api_search_collections.py index ffd84831..de546079 100644 --- a/stac_fastapi/tests/api/test_api_search_collections.py +++ b/stac_fastapi/tests/api/test_api_search_collections.py @@ -152,3 +152,77 @@ async def test_collections_fields(app_client, txn_client, load_test_data): assert "title" in collection assert "description" not in collection assert "links" in collection + + +@pytest.mark.asyncio +async def test_collections_free_text_search_get(app_client, txn_client, load_test_data): + """Verify GET /collections honors the q parameter for free text search.""" + # Create multiple collections with different content + base_collection = load_test_data("test_collection.json") + + # Use unique prefixes to avoid conflicts between tests + test_prefix = f"q-get-{uuid.uuid4().hex[:8]}" + + # Create collections with different content to test free text search + test_collections = [ + { + "id": f"{test_prefix}-sentinel", + "title": "Sentinel-2 Collection", + "description": "Collection of Sentinel-2 data", + "summaries": {"platform": ["sentinel-2a", "sentinel-2b"]}, + }, + { + "id": f"{test_prefix}-landsat", + "title": "Landsat Collection", + "description": "Collection of Landsat data", + "summaries": {"platform": ["landsat-8", "landsat-9"]}, + }, + { + "id": f"{test_prefix}-modis", + "title": "MODIS Collection", + "description": "Collection of MODIS data", + "summaries": {"platform": ["terra", "aqua"]}, + }, + ] + + for i, coll in enumerate(test_collections): + test_collection = base_collection.copy() + test_collection["id"] = coll["id"] + test_collection["title"] = coll["title"] + test_collection["description"] = coll["description"] + test_collection["summaries"] = coll["summaries"] + await create_collection(txn_client, test_collection) + + # Test free text search for "sentinel" + resp = await app_client.get( + "/collections", + params=[("q", "sentinel")], + ) + assert resp.status_code == 200 + resp_json = resp.json() + + # Filter collections to only include the ones we created for this test + found_collections = [ + c for c in resp_json["collections"] if c["id"].startswith(test_prefix) + ] + + # Should only find the sentinel collection + assert len(found_collections) == 1 + assert found_collections[0]["id"] == f"{test_prefix}-sentinel" + + # Test free text search for "landsat" + resp = await app_client.get( + "/collections", + params=[("q", "modis")], + ) + assert resp.status_code == 200 + resp_json = resp.json() + + # Filter collections to only include the ones we created for this test + found_collections = [ + c for c in resp_json["collections"] if c["id"].startswith(test_prefix) + ] + + # Should only find the landsat collection + assert len(found_collections) == 1 + assert found_collections[0]["id"] == f"{test_prefix}-modis" From a4ae47532e07432b15f382bdeadfb636a4f63aad Mon Sep 17 00:00:00 2001 From: Yuri Zmytrakov Date: Thu, 25 Sep 2025 14:09:29 +0200 Subject: [PATCH 5/7] docs: Add redis configuration documentation and changelog --- CHANGELOG.md | 2 +- README.md | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d0794819..b7e433ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] ### Added - +- Added Redis caching configuration for navigation pagination support, enabling proper `prev` and `next` links in paginated responses. [#466](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/466) - GET `/collections` collection search free text extension ex. `/collections?q=sentinel`. [#470](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/470) - Added `USE_DATETIME` environment variable to configure datetime search behavior in SFEOS. [#452](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/452) - GET `/collections` collection search sort extension ex. `/collections?sortby=+id`. [#456](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/456) diff --git a/README.md b/README.md index 78c02408..3791f163 100644 --- a/README.md +++ b/README.md @@ -284,6 +284,29 @@ You can customize additional settings in your `.env` file: > [!NOTE] > The variables `ES_HOST`, `ES_PORT`, `ES_USE_SSL`, `ES_VERIFY_CERTS` and `ES_TIMEOUT` apply to both Elasticsearch and OpenSearch backends, so there is no need to rename the key names to `OS_` even if you're using OpenSearch. +**Redis for Navigation:** +These Redis configuration variables enable proper navigation functionality in STAC FastAPI. The Redis cache stores navigation state for paginated results, allowing the system to maintain previous page links using tokens. The configuration supports either Redis Sentinel or Redis: + +| Variable | Description | Default | Required | +|------------------------------|--------------------------------------------------------------------------------------|--------------------------|---------------------------------------------------------------------------------------------| +| **Redis Sentinel** | | | | +| `REDIS_SENTINEL_HOSTS` | Comma-separated list of Redis Sentinel hostnames/IP addresses. | `""` | Conditional (required if using Sentinel) | +| `REDIS_SENTINEL_PORTS` | Comma-separated list of Redis Sentinel ports (must match order). | `"26379"` | Conditional (required if using Sentinel) | +| `REDIS_SENTINEL_MASTER_NAME` | Name of the Redis master node in Sentinel configuration. | `"master"` | Conditional (required if using Sentinel) | +| **Redis** | | | | +| `REDIS_HOST` | Redis server hostname or IP address for Redis configuration. | `""` | Conditional (required for standalone Redis) | +| `REDIS_PORT` | Redis server port for Redis configuration. | `6379` | Conditional (required for standalone Redis) | +| **Both** | | | | +| `REDIS_DB` | Redis database number to use for caching. | `0` (Sentinel) / `0` (Standalone) | Optional | +| `REDIS_MAX_CONNECTIONS` | Maximum number of connections in the Redis connection pool. | `10` | Optional | +| `REDIS_RETRY_TIMEOUT` | Enable retry on timeout for Redis operations. | `true` | Optional | +| `REDIS_DECODE_RESPONSES` | Automatically decode Redis responses to strings. | `true` | Optional | +| `REDIS_CLIENT_NAME` | Client name identifier for Redis connections. | `"stac-fastapi-app"` | Optional | +| `REDIS_HEALTH_CHECK_INTERVAL`| Interval in seconds for Redis health checks. | `30` | Optional | + +> [!NOTE] +> Use either the Sentinel configuration (`REDIS_SENTINEL_HOSTS`, `REDIS_SENTINEL_PORTS`, `REDIS_SENTINEL_MASTER_NAME`) OR the Redis configuration (`REDIS_HOST`, `REDIS_PORT`), but not both. + ## Datetime-Based Index Management ### Overview From e8d55c92c4aaee4f988b867ab877e917be1f5371 Mon Sep 17 00:00:00 2001 From: Jonathan Healy Date: Thu, 25 Sep 2025 15:20:08 +0800 Subject: [PATCH 6/7] Release v6.4.0 (#467) **Related Issue(s):** - None **Description:** - Release v6.4.0 **PR Checklist:** - [x] Code is formatted and linted (run `pre-commit run --all-files`) - [x] Tests pass (run `make test`) - [x] Documentation has been updated to reflect changes, if applicable - [x] Changes are added to the changelog --- CHANGELOG.md | 14 +++++++++++++- stac_fastapi/core/stac_fastapi/core/version.py | 2 +- stac_fastapi/elasticsearch/setup.py | 4 ++-- .../stac_fastapi/elasticsearch/version.py | 2 +- stac_fastapi/opensearch/setup.py | 4 ++-- .../opensearch/stac_fastapi/opensearch/version.py | 2 +- stac_fastapi/sfeos_helpers/setup.py | 2 +- .../stac_fastapi/sfeos_helpers/version.py | 2 +- 8 files changed, 22 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b7e433ac..c5aa732c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,18 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] ### Added + - Added Redis caching configuration for navigation pagination support, enabling proper `prev` and `next` links in paginated responses. [#466](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/466) + +### Changed + +### Fixed + + +## [v6.4.0] - 2025-09-24 + +### Added + - GET `/collections` collection search free text extension ex. `/collections?q=sentinel`. [#470](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/470) - Added `USE_DATETIME` environment variable to configure datetime search behavior in SFEOS. [#452](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/452) - GET `/collections` collection search sort extension ex. `/collections?sortby=+id`. [#456](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/456) @@ -523,7 +534,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Use genexp in execute_search and get_all_collections to return results. - Added db_to_stac serializer to item_collection method in core.py. -[Unreleased]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v6.3.0...main +[Unreleased]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v6.4.0...main +[v6.4.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v6.3.0...v6.4.0 [v6.3.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v6.2.1...v6.3.0 [v6.2.1]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v6.2.0...v6.2.1 [v6.2.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v6.1.0...v6.2.0 diff --git a/stac_fastapi/core/stac_fastapi/core/version.py b/stac_fastapi/core/stac_fastapi/core/version.py index 5777e415..111ae424 100644 --- a/stac_fastapi/core/stac_fastapi/core/version.py +++ b/stac_fastapi/core/stac_fastapi/core/version.py @@ -1,2 +1,2 @@ """library version.""" -__version__ = "6.3.0" +__version__ = "6.4.0" diff --git a/stac_fastapi/elasticsearch/setup.py b/stac_fastapi/elasticsearch/setup.py index 6977b4a4..f28b2eaf 100644 --- a/stac_fastapi/elasticsearch/setup.py +++ b/stac_fastapi/elasticsearch/setup.py @@ -6,8 +6,8 @@ desc = f.read() install_requires = [ - "stac-fastapi-core==6.3.0", - "sfeos-helpers==6.3.0", + "stac-fastapi-core==6.4.0", + "sfeos-helpers==6.4.0", "elasticsearch[async]~=8.18.0", "uvicorn~=0.23.0", "starlette>=0.35.0,<0.36.0", diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py index 5777e415..111ae424 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py @@ -1,2 +1,2 @@ """library version.""" -__version__ = "6.3.0" +__version__ = "6.4.0" diff --git a/stac_fastapi/opensearch/setup.py b/stac_fastapi/opensearch/setup.py index 3253c3aa..7dc18dff 100644 --- a/stac_fastapi/opensearch/setup.py +++ b/stac_fastapi/opensearch/setup.py @@ -6,8 +6,8 @@ desc = f.read() install_requires = [ - "stac-fastapi-core==6.3.0", - "sfeos-helpers==6.3.0", + "stac-fastapi-core==6.4.0", + "sfeos-helpers==6.4.0", "opensearch-py~=2.8.0", "opensearch-py[async]~=2.8.0", "uvicorn~=0.23.0", diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py index 5777e415..111ae424 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py @@ -1,2 +1,2 @@ """library version.""" -__version__ = "6.3.0" +__version__ = "6.4.0" diff --git a/stac_fastapi/sfeos_helpers/setup.py b/stac_fastapi/sfeos_helpers/setup.py index b5f1fbb7..5e2985b3 100644 --- a/stac_fastapi/sfeos_helpers/setup.py +++ b/stac_fastapi/sfeos_helpers/setup.py @@ -6,7 +6,7 @@ desc = f.read() install_requires = [ - "stac-fastapi.core==6.3.0", + "stac-fastapi.core==6.4.0", ] setup( diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/version.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/version.py index 5777e415..111ae424 100644 --- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/version.py +++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/version.py @@ -1,2 +1,2 @@ """library version.""" -__version__ = "6.3.0" +__version__ = "6.4.0" From 64bf85bfe139978f48fea7a88e221a83e7638c0a Mon Sep 17 00:00:00 2001 From: Yuri Zmytrakov Date: Tue, 7 Oct 2025 20:29:08 +0200 Subject: [PATCH 7/7] fix: apply recommendations --- .github/workflows/cicd.yml | 13 +++ Makefile | 28 +++++-- compose-redis.yml | 13 +++ stac_fastapi/core/stac_fastapi/core/core.py | 36 +++++---- .../core/stac_fastapi/core/redis_utils.py | 2 +- stac_fastapi/tests/redis/__init__.py | 0 .../tests/redis/test_redis_pagination.py | 80 +++++++++++++++++++ stac_fastapi/tests/redis/test_redis_utils.py | 44 ++++++++++ 8 files changed, 191 insertions(+), 25 deletions(-) create mode 100644 compose-redis.yml create mode 100644 stac_fastapi/tests/redis/__init__.py create mode 100644 stac_fastapi/tests/redis/test_redis_pagination.py create mode 100644 stac_fastapi/tests/redis/test_redis_utils.py diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 06615a7e..86ae2229 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -66,6 +66,16 @@ jobs: ports: - 9202:9202 + redis: + image: redis:7-alpine + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 6379:6379 + strategy: matrix: python-version: [ "3.9", "3.10", "3.11", "3.12", "3.13"] @@ -126,3 +136,6 @@ jobs: DATABASE_REFRESH: true ES_VERIFY_CERTS: false BACKEND: ${{ matrix.backend == 'elasticsearch7' && 'elasticsearch' || matrix.backend == 'elasticsearch8' && 'elasticsearch' || 'opensearch' }} + REDIS_ENABLE: true + REDIS_HOST: localhost + REDIS_PORT: 6379 \ No newline at end of file diff --git a/Makefile b/Makefile index 317aa630..73312ed5 100644 --- a/Makefile +++ b/Makefile @@ -63,13 +63,15 @@ docker-shell-os: .PHONY: test-elasticsearch test-elasticsearch: - -$(run_es) /bin/bash -c 'pip install redis==6.4.0 export && ./scripts/wait-for-it-es.sh elasticsearch:9200 && cd stac_fastapi/tests/ && pytest' - docker compose down + docker compose -f compose-redis.yml up -d + -$(run_es) /bin/bash -c 'export REDIS_ENABLE=true REDIS_HOST=redis REDIS_PORT=6379 && ./scripts/wait-for-it-es.sh elasticsearch:9200 && cd stac_fastapi/tests/ && pytest' + docker compose -f compose-redis.yml down -.PHONY: test-opensearch +.PHONY: test-opensearch test-opensearch: - -$(run_os) /bin/bash -c 'pip install redis==6.4.0 export && ./scripts/wait-for-it-es.sh opensearch:9202 && cd stac_fastapi/tests/ && pytest' - docker compose down + docker compose -f compose-redis.yml up -d + -$(run_os) /bin/bash -c 'export REDIS_ENABLE=true REDIS_HOST=redis REDIS_PORT=6379 && ./scripts/wait-for-it-es.sh opensearch:9202 && cd stac_fastapi/tests/ && pytest' + docker compose -f compose-redis.yml down .PHONY: test-datetime-filtering-es test-datetime-filtering-es: @@ -82,7 +84,7 @@ test-datetime-filtering-os: docker compose down .PHONY: test -test: test-elasticsearch test-datetime-filtering-es test-opensearch test-datetime-filtering-os +test: test-elasticsearch test-datetime-filtering-es test-opensearch test-datetime-filtering-os test-redis-es test-redis-os .PHONY: run-database-es run-database-es: @@ -117,4 +119,16 @@ docs-image: .PHONY: docs docs: docs-image docker compose -f compose.docs.yml \ - run docs \ No newline at end of file + run docs + +.PHONY: test-redis-es +test-redis-es: + docker compose -f compose-redis.yml up -d + -$(run_es) /bin/bash -c 'export REDIS_ENABLE=true REDIS_HOST=redis REDIS_PORT=6379 && ./scripts/wait-for-it-es.sh elasticsearch:9200 && cd stac_fastapi/tests/ && pytest redis/ -v' + docker compose -f compose-redis.yml down + +.PHONY: test-redis-os +test-redis-os: + docker compose -f compose-redis.yml up -d + -$(run_os) /bin/bash -c 'export REDIS_ENABLE=true REDIS_HOST=redis REDIS_PORT=6379 && ./scripts/wait-for-it-es.sh opensearch:9202 && cd stac_fastapi/tests/ && pytest redis/ -v' + docker compose -f compose-redis.yml down diff --git a/compose-redis.yml b/compose-redis.yml new file mode 100644 index 00000000..b572e731 --- /dev/null +++ b/compose-redis.yml @@ -0,0 +1,13 @@ +version: '3.8' + +services: + redis: + image: redis:7-alpine + ports: + - "6379:6379" + volumes: + - redis_test_data:/data + command: redis-server --appendonly yes + +volumes: + redis_test_data: diff --git a/stac_fastapi/core/stac_fastapi/core/core.py b/stac_fastapi/core/stac_fastapi/core/core.py index b6f89e64..882bbb40 100644 --- a/stac_fastapi/core/stac_fastapi/core/core.py +++ b/stac_fastapi/core/stac_fastapi/core/core.py @@ -24,14 +24,10 @@ from stac_fastapi.core.base_settings import ApiBaseSettings from stac_fastapi.core.datetime_utils import format_datetime_range from stac_fastapi.core.models.links import PagingLinks -from stac_fastapi.core.redis_utils import ( - connect_redis_sentinel, - get_prev_link, - save_self_link, -) +from stac_fastapi.core.redis_utils import connect_redis, get_prev_link, save_self_link from stac_fastapi.core.serializers import CollectionSerializer, ItemSerializer from stac_fastapi.core.session import Session -from stac_fastapi.core.utilities import filter_fields +from stac_fastapi.core.utilities import filter_fields, get_bool_env from stac_fastapi.extensions.core.transaction import AsyncBaseTransactionsClient from stac_fastapi.extensions.core.transaction.request import ( PartialCollection, @@ -277,11 +273,13 @@ async def all_collections( sort = parsed_sort current_url = str(request.url) - redis = None - try: - redis = await connect_redis_sentinel() - except Exception: - redis = None + redis_enable = get_bool_env("REDIS_ENABLE", default=False) + + if redis_enable: + try: + redis = await connect_redis() + except Exception: + redis = None # Convert q to a list if it's a string q_list = None @@ -311,7 +309,7 @@ async def all_collections( }, ] - if redis: + if redis_enable and redis: if next_token: await save_self_link(redis, next_token, current_url) @@ -557,10 +555,14 @@ async def post_search( HTTPException: If there is an error with the cql2_json filter. """ base_url = str(request.base_url) - try: - redis = await connect_redis_sentinel() - except Exception: - redis = None + redis_enable = get_bool_env("REDIS_ENABLE", default=False) + + redis = None + if redis_enable: + try: + redis = await connect_redis() + except Exception: + redis = None search = self.database.make_search() @@ -690,7 +692,7 @@ async def post_search( ) links.extend(collection_links) - if redis: + if redis_enable and redis: self_link = str(request.url) await save_self_link(redis, next_token, self_link) diff --git a/stac_fastapi/core/stac_fastapi/core/redis_utils.py b/stac_fastapi/core/stac_fastapi/core/redis_utils.py index e0ab6d11..b7923daa 100644 --- a/stac_fastapi/core/stac_fastapi/core/redis_utils.py +++ b/stac_fastapi/core/stac_fastapi/core/redis_utils.py @@ -39,7 +39,7 @@ class RedisSettings(BaseSettings): # Select the Redis or Redis Sentinel configuration -redis_settings: BaseSettings = RedisSentinelSettings() +redis_settings: BaseSettings = RedisSettings() async def connect_redis(settings: Optional[RedisSettings] = None) -> aioredis.Redis: diff --git a/stac_fastapi/tests/redis/__init__.py b/stac_fastapi/tests/redis/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/stac_fastapi/tests/redis/test_redis_pagination.py b/stac_fastapi/tests/redis/test_redis_pagination.py new file mode 100644 index 00000000..0fa30c46 --- /dev/null +++ b/stac_fastapi/tests/redis/test_redis_pagination.py @@ -0,0 +1,80 @@ +import uuid + +import pytest + +from ..conftest import create_collection, create_item + + +@pytest.mark.asyncio +async def test_search_pagination_uses_redis_cache( + app_client, txn_client, load_test_data +): + """Test Redis caching and navigation for the /search endpoint.""" + + collection = load_test_data("test_collection.json") + collection_id = f"test-pagination-collection-{uuid.uuid4()}" + collection["id"] = collection_id + await create_collection(txn_client, collection) + + for i in range(5): + item = load_test_data("test_item.json") + item["id"] = f"test-pagination-item-{uuid.uuid4()}" + item["collection"] = collection_id + await create_item(txn_client, item) + + resp = await app_client.post( + "/search", json={"collections": [collection_id], "limit": 1} + ) + resp_json = resp.json() + + next_link = next( + (link for link in resp_json["links"] if link["rel"] == "next"), None + ) + next_token = next_link["body"]["token"] + + # Expect the previous link on the second page to be retrieved from Redis cache + resp2 = await app_client.post( + "/search", + json={"collections": [collection_id], "limit": 1, "token": next_token}, + ) + resp2_json = resp2.json() + + prev_link = next( + (link for link in resp2_json["links"] if link["rel"] == "prev"), None + ) + assert prev_link is not None + + +@pytest.mark.asyncio +async def test_collections_pagination_uses_redis_cache( + app_client, txn_client, load_test_data +): + """Test Redis caching and navigation for the /collection endpoint.""" + + collection_data = load_test_data("test_collection.json") + for i in range(5): + collection = collection_data.copy() + collection["id"] = f"test-collection-pagination-{uuid.uuid4()}" + collection["title"] = f"Test Collection Pagination {i}" + await create_collection(txn_client, collection) + + resp = await app_client.get("/collections", params={"limit": 1}) + assert resp.status_code == 200 + resp1_json = resp.json() + + next_link = next( + (link for link in resp1_json["links"] if link["rel"] == "next"), None + ) + next_token = next_link["href"].split("token=")[1] + + # Expect the previous link on the second page to be retrieved from Redis cache + resp2 = await app_client.get( + "/collections", params={"limit": 1, "token": next_token} + ) + assert resp2.status_code == 200 + resp2_json = resp2.json() + + prev_link = next( + (link for link in resp2_json["links"] if link["rel"] == "prev"), None + ) + assert prev_link is not None diff --git a/stac_fastapi/tests/redis/test_redis_utils.py b/stac_fastapi/tests/redis/test_redis_utils.py new file mode 100644 index 00000000..d4e80ce5 --- /dev/null +++ b/stac_fastapi/tests/redis/test_redis_utils.py @@ -0,0 +1,44 @@ +import pytest + +from stac_fastapi.core.redis_utils import connect_redis, get_prev_link, save_self_link + + +@pytest.mark.asyncio +async def test_redis_connection(): + """Test Redis connection.""" + redis = await connect_redis() + assert redis is not None + + # Test set/get + await redis.set("string_key", "string_value") + string_value = await redis.get("string_key") + assert string_value == "string_value" + + # Test key retrieval operation + exists = await redis.exists("string_key") + assert exists == 1 + + # Test key deletion + await redis.delete("string_key") + deleted_value = await redis.get("string_key") + assert deleted_value is None + + +@pytest.mark.asyncio +async def test_redis_utils_functions(): + redis = await connect_redis() + assert redis is not None + + token = "test_token_123" + self_link = "http://mywebsite.com/search?token=test_token_123" + + await save_self_link(redis, token, self_link) + retrieved_link = await get_prev_link(redis, token) + assert retrieved_link == self_link + + await save_self_link(redis, None, "should_not_save") + null_result = await get_prev_link(redis, None) + assert null_result is None + + non_existent = await get_prev_link(redis, "non_existent_token") + assert non_existent is None