diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 06615a7e..86ae2229 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -66,6 +66,16 @@ jobs: ports: - 9202:9202 + redis: + image: redis:7-alpine + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 6379:6379 + strategy: matrix: python-version: [ "3.9", "3.10", "3.11", "3.12", "3.13"] @@ -126,3 +136,6 @@ jobs: DATABASE_REFRESH: true ES_VERIFY_CERTS: false BACKEND: ${{ matrix.backend == 'elasticsearch7' && 'elasticsearch' || matrix.backend == 'elasticsearch8' && 'elasticsearch' || 'opensearch' }} + REDIS_ENABLE: true + REDIS_HOST: localhost + REDIS_PORT: 6379 \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index e79dca0e..c5aa732c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,24 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Added +- Added Redis caching configuration for navigation pagination support, enabling proper `prev` and `next` links in paginated responses. [#466](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/466) + +### Changed + +### Fixed + + +## [v6.4.0] - 2025-09-24 + +### Added + +- GET `/collections` collection search free text extension ex. `/collections?q=sentinel`. [#470](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/470) - Added `USE_DATETIME` environment variable to configure datetime search behavior in SFEOS. [#452](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/452) - GET `/collections` collection search sort extension ex. `/collections?sortby=+id`. [#456](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/456) +- GET `/collections` collection search fields extension ex. `/collections?fields=id,title`. [#465](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/465) +- Improved error messages for sorting on unsortable fields in collection search, including guidance on how to make fields sortable. [#465](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/465) +- Added field alias for `temporal` to enable easier sorting by temporal extent, alongside `extent.temporal.interval`. [#465](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/465) +- Added `ENABLE_COLLECTIONS_SEARCH` environment variable to make collection search extensions optional (defaults to enabled). [#465](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/465) ### Changed @@ -518,7 +534,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Use genexp in execute_search and get_all_collections to return results. - Added db_to_stac serializer to item_collection method in core.py. -[Unreleased]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v6.3.0...main +[Unreleased]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v6.4.0...main +[v6.4.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v6.3.0...v6.4.0 [v6.3.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v6.2.1...v6.3.0 [v6.2.1]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v6.2.0...v6.2.1 [v6.2.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v6.1.0...v6.2.0 diff --git a/Makefile b/Makefile index 204b31a1..73312ed5 100644 --- a/Makefile +++ b/Makefile @@ -63,26 +63,28 @@ docker-shell-os: .PHONY: test-elasticsearch test-elasticsearch: - -$(run_es) /bin/bash -c 'export && ./scripts/wait-for-it-es.sh elasticsearch:9200 && cd stac_fastapi/tests/ && pytest' - docker compose down + docker compose -f compose-redis.yml up -d + -$(run_es) /bin/bash -c 'export REDIS_ENABLE=true REDIS_HOST=redis REDIS_PORT=6379 && ./scripts/wait-for-it-es.sh elasticsearch:9200 && cd stac_fastapi/tests/ && pytest' + docker compose -f compose-redis.yml down -.PHONY: test-opensearch +.PHONY: test-opensearch test-opensearch: - -$(run_os) /bin/bash -c 'export && ./scripts/wait-for-it-es.sh opensearch:9202 && cd stac_fastapi/tests/ && pytest' - docker compose down + docker compose -f compose-redis.yml up -d + -$(run_os) /bin/bash -c 'export REDIS_ENABLE=true REDIS_HOST=redis REDIS_PORT=6379 && ./scripts/wait-for-it-es.sh opensearch:9202 && cd stac_fastapi/tests/ && pytest' + docker compose -f compose-redis.yml down .PHONY: test-datetime-filtering-es test-datetime-filtering-es: - -$(run_es) /bin/bash -c 'export ENABLE_DATETIME_INDEX_FILTERING=true && ./scripts/wait-for-it-es.sh elasticsearch:9200 && cd stac_fastapi/tests/ && pytest -s --cov=stac_fastapi --cov-report=term-missing -m datetime_filtering' + -$(run_es) /bin/bash -c 'pip install redis==6.4.0 && export ENABLE_DATETIME_INDEX_FILTERING=true && ./scripts/wait-for-it-es.sh elasticsearch:9200 && cd stac_fastapi/tests/ && pytest -s --cov=stac_fastapi --cov-report=term-missing -m datetime_filtering' docker compose down .PHONY: test-datetime-filtering-os test-datetime-filtering-os: - -$(run_os) /bin/bash -c 'export ENABLE_DATETIME_INDEX_FILTERING=true && ./scripts/wait-for-it-es.sh opensearch:9202 && cd stac_fastapi/tests/ && pytest -s --cov=stac_fastapi --cov-report=term-missing -m datetime_filtering' + -$(run_os) /bin/bash -c 'pip install redis==6.4.0 && export ENABLE_DATETIME_INDEX_FILTERING=true && ./scripts/wait-for-it-es.sh opensearch:9202 && cd stac_fastapi/tests/ && pytest -s --cov=stac_fastapi --cov-report=term-missing -m datetime_filtering' docker compose down .PHONY: test -test: test-elasticsearch test-datetime-filtering-es test-opensearch test-datetime-filtering-os +test: test-elasticsearch test-datetime-filtering-es test-opensearch test-datetime-filtering-os test-redis-es test-redis-os .PHONY: run-database-es run-database-es: @@ -117,4 +119,16 @@ docs-image: .PHONY: docs docs: docs-image docker compose -f compose.docs.yml \ - run docs \ No newline at end of file + run docs + +.PHONY: test-redis-es +test-redis-es: + docker compose -f compose-redis.yml up -d + -$(run_es) /bin/bash -c 'export REDIS_ENABLE=true REDIS_HOST=redis REDIS_PORT=6379 && ./scripts/wait-for-it-es.sh elasticsearch:9200 && cd stac_fastapi/tests/ && pytest redis/ -v' + docker compose -f compose-redis.yml down + +.PHONY: test-redis-os +test-redis-os: + docker compose -f compose-redis.yml up -d + -$(run_os) /bin/bash -c 'export REDIS_ENABLE=true REDIS_HOST=redis REDIS_PORT=6379 && ./scripts/wait-for-it-es.sh opensearch:9202 && cd stac_fastapi/tests/ && pytest redis/ -v' + docker compose -f compose-redis.yml down diff --git a/README.md b/README.md index 578a440a..3791f163 100644 --- a/README.md +++ b/README.md @@ -36,11 +36,10 @@ SFEOS (stac-fastapi-elasticsearch-opensearch) is a high-performance, scalable AP - **Scale to millions of geospatial assets** with fast search performance through optimized spatial indexing and query capabilities - **Support OGC-compliant filtering** including spatial operations (intersects, contains, etc.) and temporal queries - **Perform geospatial aggregations** to analyze data distribution across space and time +- **Enhanced collection search capabilities** with support for sorting and field selection This implementation builds on the STAC-FastAPI framework, providing a production-ready solution specifically optimized for Elasticsearch and OpenSearch databases. It's ideal for organizations managing large geospatial data catalogs who need efficient discovery and access capabilities through standardized APIs. - - ## Common Deployment Patterns stac-fastapi-elasticsearch-opensearch can be deployed in several ways depending on your needs: @@ -72,6 +71,7 @@ This project is built on the following technologies: STAC, stac-fastapi, FastAPI - [Common Deployment Patterns](#common-deployment-patterns) - [Technologies](#technologies) - [Table of Contents](#table-of-contents) + - [Collection Search Extensions](#collection-search-extensions) - [Documentation \& Resources](#documentation--resources) - [Package Structure](#package-structure) - [Examples](#examples) @@ -113,6 +113,37 @@ This project is built on the following technologies: STAC, stac-fastapi, FastAPI - [Gitter Chat](https://app.gitter.im/#/room/#stac-fastapi-elasticsearch_community:gitter.im) - For real-time discussions - [GitHub Discussions](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/discussions) - For longer-form questions and answers +## Collection Search Extensions + +SFEOS implements extended capabilities for the `/collections` endpoint, allowing for more powerful collection discovery: + +- **Sorting**: Sort collections by sortable fields using the `sortby` parameter + - Example: `/collections?sortby=+id` (ascending sort by ID) + - Example: `/collections?sortby=-id` (descending sort by ID) + - Example: `/collections?sortby=-temporal` (descending sort by temporal extent) + +- **Field Selection**: Request only specific fields to be returned using the `fields` parameter + - Example: `/collections?fields=id,title,description` + - This helps reduce payload size when only certain fields are needed + +- **Free Text Search**: Search across collection text fields using the `q` parameter + - Example: `/collections?q=landsat` + - Searches across multiple text fields including title, description, and keywords + - Supports partial word matching and relevance-based sorting + +These extensions make it easier to build user interfaces that display and navigate through collections efficiently. + +> **Configuration**: Collection search extensions can be disabled by setting the `ENABLE_COLLECTIONS_SEARCH` environment variable to `false`. By default, these extensions are enabled. + +> **Note**: Sorting is only available on fields that are indexed for sorting in Elasticsearch/OpenSearch. With the default mappings, you can sort on: +> - `id` (keyword field) +> - `extent.temporal.interval` (date field) +> - `temporal` (alias to extent.temporal.interval) +> +> Text fields like `title` and `description` are not sortable by default as they use text analysis for better search capabilities. Attempting to sort on these fields will result in a user-friendly error message explaining which fields are sortable and how to make additional fields sortable by updating the mappings. +> +> **Important**: Adding keyword fields to make text fields sortable can significantly increase the index size, especially for large text fields. Consider the storage implications when deciding which fields to make sortable. + ## Package Structure This project is organized into several packages, each with a specific purpose: @@ -243,6 +274,7 @@ You can customize additional settings in your `.env` file: | `ENABLE_DIRECT_RESPONSE` | Enable direct response for maximum performance (disables all FastAPI dependencies, including authentication, custom status codes, and validation) | `false` | Optional | | `RAISE_ON_BULK_ERROR` | Controls whether bulk insert operations raise exceptions on errors. If set to `true`, the operation will stop and raise an exception when an error occurs. If set to `false`, errors will be logged, and the operation will continue. **Note:** STAC Item and ItemCollection validation errors will always raise, regardless of this flag. | `false` | Optional | | `DATABASE_REFRESH` | Controls whether database operations refresh the index immediately after changes. If set to `true`, changes will be immediately searchable. If set to `false`, changes may not be immediately visible but can improve performance for bulk operations. If set to `wait_for`, changes will wait for the next refresh cycle to become visible. | `false` | Optional | +| `ENABLE_COLLECTIONS_SEARCH` | Enable collection search extensions (sort, fields). | `true` | Optional | | `ENABLE_TRANSACTIONS_EXTENSIONS` | Enables or disables the Transactions and Bulk Transactions API extensions. If set to `false`, the POST `/collections` route and related transaction endpoints (including bulk transaction operations) will be unavailable in the API. This is useful for deployments where mutating the catalog via the API should be prevented. | `true` | Optional | | `STAC_ITEM_LIMIT` | Sets the environment variable for result limiting to SFEOS for the number of returned items and STAC collections. | `10` | Optional | | `STAC_INDEX_ASSETS` | Controls if Assets are indexed when added to Elasticsearch/Opensearch. This allows asset fields to be included in search queries. | `false` | Optional | @@ -252,6 +284,29 @@ You can customize additional settings in your `.env` file: > [!NOTE] > The variables `ES_HOST`, `ES_PORT`, `ES_USE_SSL`, `ES_VERIFY_CERTS` and `ES_TIMEOUT` apply to both Elasticsearch and OpenSearch backends, so there is no need to rename the key names to `OS_` even if you're using OpenSearch. +**Redis for Navigation:** +These Redis configuration variables enable proper navigation functionality in STAC FastAPI. The Redis cache stores navigation state for paginated results, allowing the system to maintain previous page links using tokens. The configuration supports either Redis Sentinel or Redis: + +| Variable | Description | Default | Required | +|------------------------------|--------------------------------------------------------------------------------------|--------------------------|---------------------------------------------------------------------------------------------| +| **Redis Sentinel** | | | | +| `REDIS_SENTINEL_HOSTS` | Comma-separated list of Redis Sentinel hostnames/IP addresses. | `""` | Conditional (required if using Sentinel) | +| `REDIS_SENTINEL_PORTS` | Comma-separated list of Redis Sentinel ports (must match order). | `"26379"` | Conditional (required if using Sentinel) | +| `REDIS_SENTINEL_MASTER_NAME` | Name of the Redis master node in Sentinel configuration. | `"master"` | Conditional (required if using Sentinel) | +| **Redis** | | | | +| `REDIS_HOST` | Redis server hostname or IP address for Redis configuration. | `""` | Conditional (required for standalone Redis) | +| `REDIS_PORT` | Redis server port for Redis configuration. | `6379` | Conditional (required for standalone Redis) | +| **Both** | | | | +| `REDIS_DB` | Redis database number to use for caching. | `0` (Sentinel) / `0` (Standalone) | Optional | +| `REDIS_MAX_CONNECTIONS` | Maximum number of connections in the Redis connection pool. | `10` | Optional | +| `REDIS_RETRY_TIMEOUT` | Enable retry on timeout for Redis operations. | `true` | Optional | +| `REDIS_DECODE_RESPONSES` | Automatically decode Redis responses to strings. | `true` | Optional | +| `REDIS_CLIENT_NAME` | Client name identifier for Redis connections. | `"stac-fastapi-app"` | Optional | +| `REDIS_HEALTH_CHECK_INTERVAL`| Interval in seconds for Redis health checks. | `30` | Optional | + +> [!NOTE] +> Use either the Sentinel configuration (`REDIS_SENTINEL_HOSTS`, `REDIS_SENTINEL_PORTS`, `REDIS_SENTINEL_MASTER_NAME`) OR the Redis configuration (`REDIS_HOST`, `REDIS_PORT`), but not both. + ## Datetime-Based Index Management ### Overview @@ -389,6 +444,10 @@ The system uses a precise naming convention: - **Root Path Configuration**: The application root path is the base URL by default. - For AWS Lambda with Gateway API: Set `STAC_FASTAPI_ROOT_PATH` to match the Gateway API stage name (e.g., `/v1`) +- **Feature Configuration**: Control which features are enabled: + - `ENABLE_COLLECTIONS_SEARCH`: Set to `true` (default) to enable collection search extensions (sort, fields). Set to `false` to disable. + - `ENABLE_TRANSACTIONS_EXTENSIONS`: Set to `true` (default) to enable transaction extensions. Set to `false` to disable. + ## Collection Pagination diff --git a/compose-redis.yml b/compose-redis.yml new file mode 100644 index 00000000..b572e731 --- /dev/null +++ b/compose-redis.yml @@ -0,0 +1,13 @@ +version: '3.8' + +services: + redis: + image: redis:7-alpine + ports: + - "6379:6379" + volumes: + - redis_test_data:/data + command: redis-server --appendonly yes + +volumes: + redis_test_data: diff --git a/docker-compose.redis.yml b/docker-compose.redis.yml new file mode 100644 index 00000000..14686897 --- /dev/null +++ b/docker-compose.redis.yml @@ -0,0 +1,27 @@ +version: '3.8' + +services: + redis: + image: redis:7-alpine + container_name: stac-fastapi-redis + ports: + - "6379:6379" + command: redis-server --appendonly yes + volumes: + - redis_data:/data + environment: + - REDIS_PORT=6379 + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 5s + retries: 3 + networks: + - stac-network + +volumes: + redis_data: + +networks: + stac-network: + driver: bridge \ No newline at end of file diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 00000000..a98660ec --- /dev/null +++ b/mypy.ini @@ -0,0 +1,3 @@ +[mypy] +[mypy-redis.*] +ignore_missing_imports = True \ No newline at end of file diff --git a/stac_fastapi/core/setup.py b/stac_fastapi/core/setup.py index 92442997..b055eecd 100644 --- a/stac_fastapi/core/setup.py +++ b/stac_fastapi/core/setup.py @@ -19,6 +19,7 @@ "pygeofilter~=0.3.1", "jsonschema~=4.0.0", "slowapi~=0.1.9", + "redis==6.4.0", ] setup( diff --git a/stac_fastapi/core/stac_fastapi/core/core.py b/stac_fastapi/core/stac_fastapi/core/core.py index a38bdddb..882bbb40 100644 --- a/stac_fastapi/core/stac_fastapi/core/core.py +++ b/stac_fastapi/core/stac_fastapi/core/core.py @@ -24,9 +24,10 @@ from stac_fastapi.core.base_settings import ApiBaseSettings from stac_fastapi.core.datetime_utils import format_datetime_range from stac_fastapi.core.models.links import PagingLinks +from stac_fastapi.core.redis_utils import connect_redis, get_prev_link, save_self_link from stac_fastapi.core.serializers import CollectionSerializer, ItemSerializer from stac_fastapi.core.session import Session -from stac_fastapi.core.utilities import filter_fields +from stac_fastapi.core.utilities import filter_fields, get_bool_env from stac_fastapi.extensions.core.transaction import AsyncBaseTransactionsClient from stac_fastapi.extensions.core.transaction.request import ( PartialCollection, @@ -225,11 +226,18 @@ async def landing_page(self, **kwargs) -> stac_types.LandingPage: return landing_page async def all_collections( - self, sortby: Optional[str] = None, **kwargs + self, + fields: Optional[List[str]] = None, + sortby: Optional[str] = None, + q: Optional[Union[str, List[str]]] = None, + **kwargs, ) -> stac_types.Collections: """Read all collections from the database. Args: + fields (Optional[List[str]]): Fields to include or exclude from the results. + sortby (Optional[str]): Sorting options for the results. + q (Optional[List[str]]): Free text search terms. **kwargs: Keyword arguments from the request. Returns: @@ -240,6 +248,15 @@ async def all_collections( limit = int(request.query_params.get("limit", os.getenv("STAC_ITEM_LIMIT", 10))) token = request.query_params.get("token") + # Process fields parameter for filtering collection properties + includes, excludes = set(), set() + if fields and self.extension_is_enabled("FieldsExtension"): + for field in fields: + if field[0] == "-": + excludes.add(field[1:]) + else: + includes.add(field[1:] if field[0] in "+ " else field) + sort = None if sortby: parsed_sort = [] @@ -255,10 +272,33 @@ async def all_collections( if parsed_sort: sort = parsed_sort + current_url = str(request.url) + redis_enable = get_bool_env("REDIS_ENABLE", default=False) + + if redis_enable: + try: + redis = await connect_redis() + except Exception: + redis = None + + # Convert q to a list if it's a string + q_list = None + if q is not None: + q_list = [q] if isinstance(q, str) else q + collections, next_token = await self.database.get_all_collections( - token=token, limit=limit, request=request, sort=sort + token=token, limit=limit, request=request, sort=sort, q=q_list ) + # Apply field filtering if fields parameter was provided + if fields and self.extension_is_enabled("FieldsExtension"): + filtered_collections = [ + filter_fields(collection, includes, excludes) + for collection in collections + ] + else: + filtered_collections = collections + links = [ {"rel": Relations.root.value, "type": MimeTypes.json, "href": base_url}, {"rel": Relations.parent.value, "type": MimeTypes.json, "href": base_url}, @@ -269,11 +309,27 @@ async def all_collections( }, ] + if redis_enable and redis: + if next_token: + await save_self_link(redis, next_token, current_url) + + prev_link = await get_prev_link(redis, token) + if prev_link: + links.insert( + 0, + { + "rel": "prev", + "type": "application/json", + "method": "GET", + "href": prev_link, + }, + ) + if next_token: next_link = PagingLinks(next=next_token, request=request).link_next() links.append(next_link) - return stac_types.Collections(collections=collections, links=links) + return stac_types.Collections(collections=filtered_collections, links=links) async def get_collection( self, collection_id: str, **kwargs @@ -499,6 +555,14 @@ async def post_search( HTTPException: If there is an error with the cql2_json filter. """ base_url = str(request.base_url) + redis_enable = get_bool_env("REDIS_ENABLE", default=False) + + redis = None + if redis_enable: + try: + redis = await connect_redis() + except Exception: + redis = None search = self.database.make_search() @@ -609,6 +673,41 @@ async def post_search( ] links = await PagingLinks(request=request, next=next_token).get_links() + collection_links = [] + if search_request.collections: + for collection_id in search_request.collections: + collection_links.extend( + [ + { + "rel": "collection", + "type": "application/json", + "href": urljoin(base_url, f"collections/{collection_id}"), + }, + { + "rel": "parent", + "type": "application/json", + "href": urljoin(base_url, f"collections/{collection_id}"), + }, + ] + ) + links.extend(collection_links) + + if redis_enable and redis: + self_link = str(request.url) + await save_self_link(redis, next_token, self_link) + + prev_link = await get_prev_link(redis, token_param) + if prev_link: + links.insert( + 0, + { + "rel": "prev", + "type": "application/json", + "method": "GET", + "href": prev_link, + }, + ) + return stac_types.ItemCollection( type="FeatureCollection", features=items, diff --git a/stac_fastapi/core/stac_fastapi/core/redis_utils.py b/stac_fastapi/core/stac_fastapi/core/redis_utils.py new file mode 100644 index 00000000..b7923daa --- /dev/null +++ b/stac_fastapi/core/stac_fastapi/core/redis_utils.py @@ -0,0 +1,124 @@ +"""Utilities for connecting to and managing Redis connections.""" + +from typing import Optional + +from pydantic_settings import BaseSettings +from redis import asyncio as aioredis +from redis.asyncio.sentinel import Sentinel + +redis_pool: Optional[aioredis.Redis] = None + + +class RedisSentinelSettings(BaseSettings): + """Configuration for connecting to Redis Sentinel.""" + + REDIS_SENTINEL_HOSTS: str = "" + REDIS_SENTINEL_PORTS: str = "26379" + REDIS_SENTINEL_MASTER_NAME: str = "master" + REDIS_DB: int = 15 + + REDIS_MAX_CONNECTIONS: int = 10 + REDIS_RETRY_TIMEOUT: bool = True + REDIS_DECODE_RESPONSES: bool = True + REDIS_CLIENT_NAME: str = "stac-fastapi-app" + REDIS_HEALTH_CHECK_INTERVAL: int = 30 + + +class RedisSettings(BaseSettings): + """Configuration for connecting Redis Sentinel.""" + + REDIS_HOST: str = "" + REDIS_PORT: int = 6379 + REDIS_DB: int = 0 + + REDIS_MAX_CONNECTIONS: int = 10 + REDIS_RETRY_TIMEOUT: bool = True + REDIS_DECODE_RESPONSES: bool = True + REDIS_CLIENT_NAME: str = "stac-fastapi-app" + REDIS_HEALTH_CHECK_INTERVAL: int = 30 + + +# Select the Redis or Redis Sentinel configuration +redis_settings: BaseSettings = RedisSettings() + + +async def connect_redis(settings: Optional[RedisSettings] = None) -> aioredis.Redis: + """Return a Redis connection.""" + global redis_pool + settings = settings or redis_settings + + if not settings.REDIS_HOST or not settings.REDIS_PORT: + return None + + if redis_pool is None: + pool = aioredis.ConnectionPool( + host=settings.REDIS_HOST, + port=settings.REDIS_PORT, + db=settings.REDIS_DB, + max_connections=settings.REDIS_MAX_CONNECTIONS, + decode_responses=settings.REDIS_DECODE_RESPONSES, + retry_on_timeout=settings.REDIS_RETRY_TIMEOUT, + health_check_interval=settings.REDIS_HEALTH_CHECK_INTERVAL, + ) + redis_pool = aioredis.Redis( + connection_pool=pool, client_name=settings.REDIS_CLIENT_NAME + ) + return redis_pool + + +async def connect_redis_sentinel( + settings: Optional[RedisSentinelSettings] = None, +) -> Optional[aioredis.Redis]: + """Return a Redis Sentinel connection.""" + global redis_pool + + settings = settings or redis_settings + + if ( + not settings.REDIS_SENTINEL_HOSTS + or not settings.REDIS_SENTINEL_PORTS + or not settings.REDIS_SENTINEL_MASTER_NAME + ): + return None + + hosts = [h.strip() for h in settings.REDIS_SENTINEL_HOSTS.split(",") if h.strip()] + ports = [ + int(p.strip()) for p in settings.REDIS_SENTINEL_PORTS.split(",") if p.strip() + ] + + if redis_pool is None: + try: + sentinel = Sentinel( + [(h, p) for h, p in zip(hosts, ports)], + decode_responses=settings.REDIS_DECODE_RESPONSES, + ) + master = sentinel.master_for( + service_name=settings.REDIS_SENTINEL_MASTER_NAME, + db=settings.REDIS_DB, + decode_responses=settings.REDIS_DECODE_RESPONSES, + retry_on_timeout=settings.REDIS_RETRY_TIMEOUT, + client_name=settings.REDIS_CLIENT_NAME, + max_connections=settings.REDIS_MAX_CONNECTIONS, + health_check_interval=settings.REDIS_HEALTH_CHECK_INTERVAL, + ) + redis_pool = master + + except Exception: + return None + + return redis_pool + + +async def save_self_link( + redis: aioredis.Redis, token: Optional[str], self_href: str +) -> None: + """Save the self link for the current token with 30 min TTL.""" + if token: + await redis.setex(f"nav:self:{token}", 1800, self_href) + + +async def get_prev_link(redis: aioredis.Redis, token: Optional[str]) -> Optional[str]: + """Get the previous page link for the current token (if exists).""" + if not token: + return None + return await redis.get(f"nav:self:{token}") diff --git a/stac_fastapi/core/stac_fastapi/core/version.py b/stac_fastapi/core/stac_fastapi/core/version.py index 5777e415..111ae424 100644 --- a/stac_fastapi/core/stac_fastapi/core/version.py +++ b/stac_fastapi/core/stac_fastapi/core/version.py @@ -1,2 +1,2 @@ """library version.""" -__version__ = "6.3.0" +__version__ = "6.4.0" diff --git a/stac_fastapi/elasticsearch/setup.py b/stac_fastapi/elasticsearch/setup.py index 6977b4a4..f28b2eaf 100644 --- a/stac_fastapi/elasticsearch/setup.py +++ b/stac_fastapi/elasticsearch/setup.py @@ -6,8 +6,8 @@ desc = f.read() install_requires = [ - "stac-fastapi-core==6.3.0", - "sfeos-helpers==6.3.0", + "stac-fastapi-core==6.4.0", + "sfeos-helpers==6.4.0", "elasticsearch[async]~=8.18.0", "uvicorn~=0.23.0", "starlette>=0.35.0,<0.36.0", diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py index ea26c022..18b52147 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py @@ -45,8 +45,7 @@ ) from stac_fastapi.extensions.core.fields import FieldsConformanceClasses from stac_fastapi.extensions.core.filter import FilterConformanceClasses - -# from stac_fastapi.extensions.core.free_text import FreeTextConformanceClasses +from stac_fastapi.extensions.core.free_text import FreeTextConformanceClasses from stac_fastapi.extensions.core.query import QueryConformanceClasses from stac_fastapi.extensions.core.sort import SortConformanceClasses from stac_fastapi.extensions.third_party import BulkTransactionExtension @@ -57,7 +56,9 @@ logger = logging.getLogger(__name__) TRANSACTIONS_EXTENSIONS = get_bool_env("ENABLE_TRANSACTIONS_EXTENSIONS", default=True) +ENABLE_COLLECTIONS_SEARCH = get_bool_env("ENABLE_COLLECTIONS_SEARCH", default=True) logger.info("TRANSACTIONS_EXTENSIONS is set to %s", TRANSACTIONS_EXTENSIONS) +logger.info("ENABLE_COLLECTIONS_SEARCH is set to %s", ENABLE_COLLECTIONS_SEARCH) settings = ElasticsearchSettings() session = Session.create_from_settings(settings) @@ -115,25 +116,26 @@ extensions = [aggregation_extension] + search_extensions -# Create collection search extensions -# Only sort extension is enabled for now -collection_search_extensions = [ - # QueryExtension(conformance_classes=[QueryConformanceClasses.COLLECTIONS]), - SortExtension(conformance_classes=[SortConformanceClasses.COLLECTIONS]), - # FieldsExtension(conformance_classes=[FieldsConformanceClasses.COLLECTIONS]), - # CollectionSearchFilterExtension( - # conformance_classes=[FilterConformanceClasses.COLLECTIONS] - # ), - # FreeTextExtension(conformance_classes=[FreeTextConformanceClasses.COLLECTIONS]), -] - -# Initialize collection search with its extensions -collection_search_ext = CollectionSearchExtension.from_extensions( - collection_search_extensions -) -collections_get_request_model = collection_search_ext.GET +# Create collection search extensions if enabled +if ENABLE_COLLECTIONS_SEARCH: + # Create collection search extensions + collection_search_extensions = [ + # QueryExtension(conformance_classes=[QueryConformanceClasses.COLLECTIONS]), + SortExtension(conformance_classes=[SortConformanceClasses.COLLECTIONS]), + FieldsExtension(conformance_classes=[FieldsConformanceClasses.COLLECTIONS]), + # CollectionSearchFilterExtension( + # conformance_classes=[FilterConformanceClasses.COLLECTIONS] + # ), + FreeTextExtension(conformance_classes=[FreeTextConformanceClasses.COLLECTIONS]), + ] + + # Initialize collection search with its extensions + collection_search_ext = CollectionSearchExtension.from_extensions( + collection_search_extensions + ) + collections_get_request_model = collection_search_ext.GET -extensions.append(collection_search_ext) + extensions.append(collection_search_ext) database_logic.extensions = [type(ext).__name__ for ext in extensions] @@ -170,10 +172,13 @@ "search_get_request_model": create_get_request_model(search_extensions), "search_post_request_model": post_request_model, "items_get_request_model": items_get_request_model, - "collections_get_request_model": collections_get_request_model, "route_dependencies": get_route_dependencies(), } +# Add collections_get_request_model if collection search is enabled +if ENABLE_COLLECTIONS_SEARCH: + app_config["collections_get_request_model"] = collections_get_request_model + api = StacApi(**app_config) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index df1e816d..c472039b 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -175,6 +175,7 @@ async def get_all_collections( limit: int, request: Request, sort: Optional[List[Dict[str, Any]]] = None, + q: Optional[List[str]] = None, ) -> Tuple[List[Dict[str, Any]], Optional[str]]: """Retrieve a list of collections from Elasticsearch, supporting pagination. @@ -183,16 +184,32 @@ async def get_all_collections( limit (int): The number of results to return. request (Request): The FastAPI request object. sort (Optional[List[Dict[str, Any]]]): Optional sort parameter from the request. + q (Optional[List[str]]): Free text search terms. Returns: A tuple of (collections, next pagination token if any). + + Raises: + HTTPException: If sorting is requested on a field that is not sortable. """ + # Define sortable fields based on the ES_COLLECTIONS_MAPPINGS + sortable_fields = ["id", "extent.temporal.interval", "temporal"] + + # Format the sort parameter formatted_sort = [] if sort: for item in sort: field = item.get("field") direction = item.get("direction", "asc") if field: + # Validate that the field is sortable + if field not in sortable_fields: + raise HTTPException( + status_code=400, + detail=f"Field '{field}' is not sortable. Sortable fields are: {', '.join(sortable_fields)}. " + + "Text fields are not sortable by default in Elasticsearch. " + + "To make a field sortable, update the mapping to use 'keyword' type or add a '.keyword' subfield. ", + ) formatted_sort.append({field: {"order": direction}}) # Always include id as a secondary sort to ensure consistent pagination if not any("id" in item for item in formatted_sort): @@ -208,6 +225,38 @@ async def get_all_collections( if token: body["search_after"] = [token] + # Apply free text query if provided + if q: + # For collections, we want to search across all relevant fields + should_clauses = [] + + # For each search term + for term in q: + # Create a multi_match query for each term + for field in [ + "id", + "title", + "description", + "keywords", + "summaries.platform", + "summaries.constellation", + "providers.name", + "providers.url", + ]: + should_clauses.append( + { + "wildcard": { + field: {"value": f"*{term}*", "case_insensitive": True} + } + } + ) + + # Add the query to the body using bool query with should clauses + body["query"] = { + "bool": {"should": should_clauses, "minimum_should_match": 1} + } + + # Execute the search response = await self.client.search( index=COLLECTIONS_INDEX, body=body, diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py index 5777e415..111ae424 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py @@ -1,2 +1,2 @@ """library version.""" -__version__ = "6.3.0" +__version__ = "6.4.0" diff --git a/stac_fastapi/opensearch/setup.py b/stac_fastapi/opensearch/setup.py index 3253c3aa..7dc18dff 100644 --- a/stac_fastapi/opensearch/setup.py +++ b/stac_fastapi/opensearch/setup.py @@ -6,8 +6,8 @@ desc = f.read() install_requires = [ - "stac-fastapi-core==6.3.0", - "sfeos-helpers==6.3.0", + "stac-fastapi-core==6.4.0", + "sfeos-helpers==6.4.0", "opensearch-py~=2.8.0", "opensearch-py[async]~=2.8.0", "uvicorn~=0.23.0", diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py index 789cb728..34d55589 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py @@ -39,8 +39,7 @@ ) from stac_fastapi.extensions.core.fields import FieldsConformanceClasses from stac_fastapi.extensions.core.filter import FilterConformanceClasses - -# from stac_fastapi.extensions.core.free_text import FreeTextConformanceClasses +from stac_fastapi.extensions.core.free_text import FreeTextConformanceClasses from stac_fastapi.extensions.core.query import QueryConformanceClasses from stac_fastapi.extensions.core.sort import SortConformanceClasses from stac_fastapi.extensions.third_party import BulkTransactionExtension @@ -57,7 +56,9 @@ logger = logging.getLogger(__name__) TRANSACTIONS_EXTENSIONS = get_bool_env("ENABLE_TRANSACTIONS_EXTENSIONS", default=True) +ENABLE_COLLECTIONS_SEARCH = get_bool_env("ENABLE_COLLECTIONS_SEARCH", default=True) logger.info("TRANSACTIONS_EXTENSIONS is set to %s", TRANSACTIONS_EXTENSIONS) +logger.info("ENABLE_COLLECTIONS_SEARCH is set to %s", ENABLE_COLLECTIONS_SEARCH) settings = OpensearchSettings() session = Session.create_from_settings(settings) @@ -115,25 +116,26 @@ extensions = [aggregation_extension] + search_extensions -# Create collection search extensions -# Only sort extension is enabled for now -collection_search_extensions = [ - # QueryExtension(conformance_classes=[QueryConformanceClasses.COLLECTIONS]), - SortExtension(conformance_classes=[SortConformanceClasses.COLLECTIONS]), - # FieldsExtension(conformance_classes=[FieldsConformanceClasses.COLLECTIONS]), - # CollectionSearchFilterExtension( - # conformance_classes=[FilterConformanceClasses.COLLECTIONS] - # ), - # FreeTextExtension(conformance_classes=[FreeTextConformanceClasses.COLLECTIONS]), -] - -# Initialize collection search with its extensions -collection_search_ext = CollectionSearchExtension.from_extensions( - collection_search_extensions -) -collections_get_request_model = collection_search_ext.GET +# Create collection search extensions if enabled +if ENABLE_COLLECTIONS_SEARCH: + # Create collection search extensions + collection_search_extensions = [ + # QueryExtension(conformance_classes=[QueryConformanceClasses.COLLECTIONS]), + SortExtension(conformance_classes=[SortConformanceClasses.COLLECTIONS]), + FieldsExtension(conformance_classes=[FieldsConformanceClasses.COLLECTIONS]), + # CollectionSearchFilterExtension( + # conformance_classes=[FilterConformanceClasses.COLLECTIONS] + # ), + FreeTextExtension(conformance_classes=[FreeTextConformanceClasses.COLLECTIONS]), + ] + + # Initialize collection search with its extensions + collection_search_ext = CollectionSearchExtension.from_extensions( + collection_search_extensions + ) + collections_get_request_model = collection_search_ext.GET -extensions.append(collection_search_ext) + extensions.append(collection_search_ext) database_logic.extensions = [type(ext).__name__ for ext in extensions] @@ -167,13 +169,16 @@ post_request_model=post_request_model, landing_page_id=os.getenv("STAC_FASTAPI_LANDING_PAGE_ID", "stac-fastapi"), ), - "collections_get_request_model": collections_get_request_model, "search_get_request_model": create_get_request_model(search_extensions), "search_post_request_model": post_request_model, "items_get_request_model": items_get_request_model, "route_dependencies": get_route_dependencies(), } +# Add collections_get_request_model if collection search is enabled +if ENABLE_COLLECTIONS_SEARCH: + app_config["collections_get_request_model"] = collections_get_request_model + api = StacApi(**app_config) diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py index 4253a00a..f4b8abd0 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py @@ -159,6 +159,7 @@ async def get_all_collections( limit: int, request: Request, sort: Optional[List[Dict[str, Any]]] = None, + q: Optional[List[str]] = None, ) -> Tuple[List[Dict[str, Any]], Optional[str]]: """Retrieve a list of collections from Elasticsearch, supporting pagination. @@ -167,16 +168,32 @@ async def get_all_collections( limit (int): The number of results to return. request (Request): The FastAPI request object. sort (Optional[List[Dict[str, Any]]]): Optional sort parameter from the request. + q (Optional[List[str]]): Free text search terms. Returns: A tuple of (collections, next pagination token if any). + + Raises: + HTTPException: If sorting is requested on a field that is not sortable. """ + # Define sortable fields based on the ES_COLLECTIONS_MAPPINGS + sortable_fields = ["id", "extent.temporal.interval", "temporal"] + + # Format the sort parameter formatted_sort = [] if sort: for item in sort: field = item.get("field") direction = item.get("direction", "asc") if field: + # Validate that the field is sortable + if field not in sortable_fields: + raise HTTPException( + status_code=400, + detail=f"Field '{field}' is not sortable. Sortable fields are: {', '.join(sortable_fields)}. " + + "Text fields are not sortable by default in OpenSearch. " + + "To make a field sortable, update the mapping to use 'keyword' type or add a '.keyword' subfield. ", + ) formatted_sort.append({field: {"order": direction}}) # Always include id as a secondary sort to ensure consistent pagination if not any("id" in item for item in formatted_sort): @@ -192,6 +209,37 @@ async def get_all_collections( if token: body["search_after"] = [token] + # Apply free text query if provided + if q: + # For collections, we want to search across all relevant fields + should_clauses = [] + + # For each search term + for term in q: + # Create a multi_match query for each term + for field in [ + "id", + "title", + "description", + "keywords", + "summaries.platform", + "summaries.constellation", + "providers.name", + "providers.url", + ]: + should_clauses.append( + { + "wildcard": { + field: {"value": f"*{term}*", "case_insensitive": True} + } + } + ) + + # Add the query to the body using bool query with should clauses + body["query"] = { + "bool": {"should": should_clauses, "minimum_should_match": 1} + } + response = await self.client.search( index=COLLECTIONS_INDEX, body=body, diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py index 5777e415..111ae424 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py @@ -1,2 +1,2 @@ """library version.""" -__version__ = "6.3.0" +__version__ = "6.4.0" diff --git a/stac_fastapi/sfeos_helpers/setup.py b/stac_fastapi/sfeos_helpers/setup.py index b5f1fbb7..5e2985b3 100644 --- a/stac_fastapi/sfeos_helpers/setup.py +++ b/stac_fastapi/sfeos_helpers/setup.py @@ -6,7 +6,7 @@ desc = f.read() install_requires = [ - "stac-fastapi.core==6.3.0", + "stac-fastapi.core==6.4.0", ] setup( diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/mappings.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/mappings.py index 17cdd1ea..df002dc5 100644 --- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/mappings.py +++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/mappings.py @@ -165,6 +165,8 @@ class Geometry(Protocol): # noqa "providers": {"type": "object", "enabled": False}, "links": {"type": "object", "enabled": False}, "item_assets": {"type": "object", "enabled": get_bool_env("STAC_INDEX_ASSETS")}, + # Field alias to allow sorting on 'temporal' (points to extent.temporal.interval) + "temporal": {"type": "alias", "path": "extent.temporal.interval"}, }, } diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/version.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/version.py index 5777e415..111ae424 100644 --- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/version.py +++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/version.py @@ -1,2 +1,2 @@ """library version.""" -__version__ = "6.3.0" +__version__ = "6.4.0" diff --git a/stac_fastapi/tests/api/test_api_search_collections.py b/stac_fastapi/tests/api/test_api_search_collections.py index ed0dfc1b..de546079 100644 --- a/stac_fastapi/tests/api/test_api_search_collections.py +++ b/stac_fastapi/tests/api/test_api_search_collections.py @@ -77,3 +77,152 @@ async def test_collections_sort_id_desc(app_client, txn_client, load_test_data): assert len(test_collections) == len(collection_ids) for i, expected_id in enumerate(sorted_ids): assert test_collections[i]["id"] == expected_id + + +@pytest.mark.asyncio +async def test_collections_fields(app_client, txn_client, load_test_data): + """Verify GET /collections honors the fields parameter.""" + # Create multiple collections with different ids + base_collection = load_test_data("test_collection.json") + + # Create collections with ids in a specific order to test fields + # Use unique prefixes to avoid conflicts between tests + test_prefix = f"fields-{uuid.uuid4().hex[:8]}" + collection_ids = [f"{test_prefix}-a", f"{test_prefix}-b", f"{test_prefix}-c"] + + for i, coll_id in enumerate(collection_ids): + test_collection = base_collection.copy() + test_collection["id"] = coll_id + test_collection["title"] = f"Test Collection {i}" + test_collection["description"] = f"Description for collection {i}" + await create_collection(txn_client, test_collection) + + # Test include fields parameter + resp = await app_client.get( + "/collections", + params=[("fields", "id"), ("fields", "title")], + ) + assert resp.status_code == 200 + resp_json = resp.json() + + # Check if collections exist in the response + assert "collections" in resp_json, "No collections in response" + + # Filter collections to only include the ones we created for this test + test_collections = [] + for c in resp_json["collections"]: + if "id" in c and c["id"].startswith(test_prefix): + test_collections.append(c) + + # Filter collections to only include the ones we created for this test + test_collections = [] + for c in resp_json["collections"]: + if "id" in c and c["id"].startswith(test_prefix): + test_collections.append(c) + + # Collections should only have id and title fields + for collection in test_collections: + assert "id" in collection + assert "title" in collection + assert "description" not in collection + assert "links" in collection # links are always included + + # Test exclude fields parameter + resp = await app_client.get( + "/collections", + params=[("fields", "-description")], + ) + assert resp.status_code == 200 + resp_json = resp.json() + + # Check if collections exist in the response + assert ( + "collections" in resp_json + ), "No collections in response for exclude fields test" + + # Filter collections to only include the ones we created for this test + test_collections = [] + for c in resp_json["collections"]: + if "id" in c and c["id"].startswith(test_prefix): + test_collections.append(c) + + # Collections should have all fields except description + for collection in test_collections: + assert "id" in collection + assert "title" in collection + assert "description" not in collection + assert "links" in collection + + +@pytest.mark.asyncio +async def test_collections_free_text_search_get(app_client, txn_client, load_test_data): + """Verify GET /collections honors the q parameter for free text search.""" + # Create multiple collections with different content + base_collection = load_test_data("test_collection.json") + + # Use unique prefixes to avoid conflicts between tests + test_prefix = f"q-get-{uuid.uuid4().hex[:8]}" + + # Create collections with different content to test free text search + test_collections = [ + { + "id": f"{test_prefix}-sentinel", + "title": "Sentinel-2 Collection", + "description": "Collection of Sentinel-2 data", + "summaries": {"platform": ["sentinel-2a", "sentinel-2b"]}, + }, + { + "id": f"{test_prefix}-landsat", + "title": "Landsat Collection", + "description": "Collection of Landsat data", + "summaries": {"platform": ["landsat-8", "landsat-9"]}, + }, + { + "id": f"{test_prefix}-modis", + "title": "MODIS Collection", + "description": "Collection of MODIS data", + "summaries": {"platform": ["terra", "aqua"]}, + }, + ] + + for i, coll in enumerate(test_collections): + test_collection = base_collection.copy() + test_collection["id"] = coll["id"] + test_collection["title"] = coll["title"] + test_collection["description"] = coll["description"] + test_collection["summaries"] = coll["summaries"] + await create_collection(txn_client, test_collection) + + # Test free text search for "sentinel" + resp = await app_client.get( + "/collections", + params=[("q", "sentinel")], + ) + assert resp.status_code == 200 + resp_json = resp.json() + + # Filter collections to only include the ones we created for this test + found_collections = [ + c for c in resp_json["collections"] if c["id"].startswith(test_prefix) + ] + + # Should only find the sentinel collection + assert len(found_collections) == 1 + assert found_collections[0]["id"] == f"{test_prefix}-sentinel" + + # Test free text search for "landsat" + resp = await app_client.get( + "/collections", + params=[("q", "modis")], + ) + assert resp.status_code == 200 + resp_json = resp.json() + + # Filter collections to only include the ones we created for this test + found_collections = [ + c for c in resp_json["collections"] if c["id"].startswith(test_prefix) + ] + + # Should only find the landsat collection + assert len(found_collections) == 1 + assert found_collections[0]["id"] == f"{test_prefix}-modis" diff --git a/stac_fastapi/tests/api/test_collections_search_env.py b/stac_fastapi/tests/api/test_collections_search_env.py new file mode 100644 index 00000000..5358faf9 --- /dev/null +++ b/stac_fastapi/tests/api/test_collections_search_env.py @@ -0,0 +1,167 @@ +"""Test the ENABLE_COLLECTIONS_SEARCH environment variable.""" + +import os +import uuid +from unittest import mock + +import pytest + +from ..conftest import create_collection, refresh_indices + + +@pytest.mark.asyncio +@mock.patch.dict(os.environ, {"ENABLE_COLLECTIONS_SEARCH": "false"}) +async def test_collections_search_disabled(app_client, txn_client, load_test_data): + """Test that collection search extensions are disabled when ENABLE_COLLECTIONS_SEARCH=false.""" + # Create multiple collections with different ids to test sorting + base_collection = load_test_data("test_collection.json") + + # Use unique prefixes to avoid conflicts between tests + test_prefix = f"disabled-{uuid.uuid4().hex[:8]}" + collection_ids = [f"{test_prefix}-c", f"{test_prefix}-a", f"{test_prefix}-b"] + + for i, coll_id in enumerate(collection_ids): + test_collection = base_collection.copy() + test_collection["id"] = coll_id + test_collection["title"] = f"Test Collection {i}" + await create_collection(txn_client, test_collection) + + # Refresh indices to ensure collections are searchable + await refresh_indices(txn_client) + + # When collection search is disabled, sortby parameter should be ignored + resp = await app_client.get( + "/collections", + params=[("sortby", "+id")], + ) + assert resp.status_code == 200 + + # Verify that results are NOT sorted by id (should be in insertion order or default order) + resp_json = resp.json() + collections = [ + c for c in resp_json["collections"] if c["id"].startswith(test_prefix) + ] + + # Extract the ids in the order they were returned + returned_ids = [c["id"] for c in collections] + + # If sorting was working, they would be in alphabetical order: a, b, c + # But since sorting is disabled, they should be in a different order + # We can't guarantee the exact order, but we can check they're not in alphabetical order + sorted_ids = sorted(returned_ids) + assert ( + returned_ids != sorted_ids or len(collections) < 2 + ), "Collections appear to be sorted despite ENABLE_COLLECTIONS_SEARCH=false" + + # Fields parameter should also be ignored + resp = await app_client.get( + "/collections", + params=[("fields", "id")], # Request only id field + ) + assert resp.status_code == 200 + + # Verify that all fields are still returned, not just id + resp_json = resp.json() + for collection in resp_json["collections"]: + if collection["id"].startswith(test_prefix): + # If fields filtering was working, only id would be present + # Since it's disabled, other fields like title should still be present + assert ( + "title" in collection + ), "Fields filtering appears to be working despite ENABLE_COLLECTIONS_SEARCH=false" + + +@pytest.mark.asyncio +@mock.patch.dict(os.environ, {"ENABLE_COLLECTIONS_SEARCH": "true"}) +async def test_collections_search_enabled(app_client, txn_client, load_test_data): + """Test that collection search extensions work when ENABLE_COLLECTIONS_SEARCH=true.""" + # Create multiple collections with different ids to test sorting + base_collection = load_test_data("test_collection.json") + + # Use unique prefixes to avoid conflicts between tests + test_prefix = f"enabled-{uuid.uuid4().hex[:8]}" + collection_ids = [f"{test_prefix}-c", f"{test_prefix}-a", f"{test_prefix}-b"] + + for i, coll_id in enumerate(collection_ids): + test_collection = base_collection.copy() + test_collection["id"] = coll_id + test_collection["title"] = f"Test Collection {i}" + await create_collection(txn_client, test_collection) + + # Refresh indices to ensure collections are searchable + await refresh_indices(txn_client) + + # Test that sortby parameter works - sort by id ascending + resp = await app_client.get( + "/collections", + params=[("sortby", "+id")], + ) + assert resp.status_code == 200 + + # Verify that results are sorted by id in ascending order + resp_json = resp.json() + collections = [ + c for c in resp_json["collections"] if c["id"].startswith(test_prefix) + ] + + # Extract the ids in the order they were returned + returned_ids = [c["id"] for c in collections] + + # Verify they're in ascending order + assert returned_ids == sorted( + returned_ids + ), "Collections are not sorted by id ascending" + + # Test that sortby parameter works - sort by id descending + resp = await app_client.get( + "/collections", + params=[("sortby", "-id")], + ) + assert resp.status_code == 200 + + # Verify that results are sorted by id in descending order + resp_json = resp.json() + collections = [ + c for c in resp_json["collections"] if c["id"].startswith(test_prefix) + ] + + # Extract the ids in the order they were returned + returned_ids = [c["id"] for c in collections] + + # Verify they're in descending order + assert returned_ids == sorted( + returned_ids, reverse=True + ), "Collections are not sorted by id descending" + + # Test that fields parameter works - request only id field + resp = await app_client.get( + "/collections", + params=[("fields", "id")], + ) + assert resp.status_code == 200 + resp_json = resp.json() + + # When fields=id is specified, collections should only have id field + for collection in resp_json["collections"]: + if collection["id"].startswith(test_prefix): + assert "id" in collection, "id field is missing" + assert ( + "title" not in collection + ), "title field should be excluded when fields=id" + + # Test that fields parameter works - request multiple fields + resp = await app_client.get( + "/collections", + params=[("fields", "id,title")], + ) + assert resp.status_code == 200 + resp_json = resp.json() + + # When fields=id,title is specified, collections should have both fields but not others + for collection in resp_json["collections"]: + if collection["id"].startswith(test_prefix): + assert "id" in collection, "id field is missing" + assert "title" in collection, "title field is missing" + assert ( + "description" not in collection + ), "description field should be excluded when fields=id,title" diff --git a/stac_fastapi/tests/redis/__init__.py b/stac_fastapi/tests/redis/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/stac_fastapi/tests/redis/test_redis_pagination.py b/stac_fastapi/tests/redis/test_redis_pagination.py new file mode 100644 index 00000000..0fa30c46 --- /dev/null +++ b/stac_fastapi/tests/redis/test_redis_pagination.py @@ -0,0 +1,80 @@ +import uuid + +import pytest + +from ..conftest import create_collection, create_item + + +@pytest.mark.asyncio +async def test_search_pagination_uses_redis_cache( + app_client, txn_client, load_test_data +): + """Test Redis caching and navigation for the /search endpoint.""" + + collection = load_test_data("test_collection.json") + collection_id = f"test-pagination-collection-{uuid.uuid4()}" + collection["id"] = collection_id + await create_collection(txn_client, collection) + + for i in range(5): + item = load_test_data("test_item.json") + item["id"] = f"test-pagination-item-{uuid.uuid4()}" + item["collection"] = collection_id + await create_item(txn_client, item) + + resp = await app_client.post( + "/search", json={"collections": [collection_id], "limit": 1} + ) + resp_json = resp.json() + + next_link = next( + (link for link in resp_json["links"] if link["rel"] == "next"), None + ) + next_token = next_link["body"]["token"] + + # Expect the previous link on the second page to be retrieved from Redis cache + resp2 = await app_client.post( + "/search", + json={"collections": [collection_id], "limit": 1, "token": next_token}, + ) + resp2_json = resp2.json() + + prev_link = next( + (link for link in resp2_json["links"] if link["rel"] == "prev"), None + ) + assert prev_link is not None + + +@pytest.mark.asyncio +async def test_collections_pagination_uses_redis_cache( + app_client, txn_client, load_test_data +): + """Test Redis caching and navigation for the /collection endpoint.""" + + collection_data = load_test_data("test_collection.json") + for i in range(5): + collection = collection_data.copy() + collection["id"] = f"test-collection-pagination-{uuid.uuid4()}" + collection["title"] = f"Test Collection Pagination {i}" + await create_collection(txn_client, collection) + + resp = await app_client.get("/collections", params={"limit": 1}) + assert resp.status_code == 200 + resp1_json = resp.json() + + next_link = next( + (link for link in resp1_json["links"] if link["rel"] == "next"), None + ) + next_token = next_link["href"].split("token=")[1] + + # Expect the previous link on the second page to be retrieved from Redis cache + resp2 = await app_client.get( + "/collections", params={"limit": 1, "token": next_token} + ) + assert resp2.status_code == 200 + resp2_json = resp2.json() + + prev_link = next( + (link for link in resp2_json["links"] if link["rel"] == "prev"), None + ) + assert prev_link is not None diff --git a/stac_fastapi/tests/redis/test_redis_utils.py b/stac_fastapi/tests/redis/test_redis_utils.py new file mode 100644 index 00000000..d4e80ce5 --- /dev/null +++ b/stac_fastapi/tests/redis/test_redis_utils.py @@ -0,0 +1,44 @@ +import pytest + +from stac_fastapi.core.redis_utils import connect_redis, get_prev_link, save_self_link + + +@pytest.mark.asyncio +async def test_redis_connection(): + """Test Redis connection.""" + redis = await connect_redis() + assert redis is not None + + # Test set/get + await redis.set("string_key", "string_value") + string_value = await redis.get("string_key") + assert string_value == "string_value" + + # Test key retrieval operation + exists = await redis.exists("string_key") + assert exists == 1 + + # Test key deletion + await redis.delete("string_key") + deleted_value = await redis.get("string_key") + assert deleted_value is None + + +@pytest.mark.asyncio +async def test_redis_utils_functions(): + redis = await connect_redis() + assert redis is not None + + token = "test_token_123" + self_link = "http://mywebsite.com/search?token=test_token_123" + + await save_self_link(redis, token, self_link) + retrieved_link = await get_prev_link(redis, token) + assert retrieved_link == self_link + + await save_self_link(redis, None, "should_not_save") + null_result = await get_prev_link(redis, None) + assert null_result is None + + non_existent = await get_prev_link(redis, "non_existent_token") + assert non_existent is None