diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 06615a7e..b639133d 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -66,6 +66,16 @@ jobs: ports: - 9202:9202 + redis: + image: redis:7-alpine + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 6379:6379 + strategy: matrix: python-version: [ "3.9", "3.10", "3.11", "3.12", "3.13"] @@ -126,3 +136,6 @@ jobs: DATABASE_REFRESH: true ES_VERIFY_CERTS: false BACKEND: ${{ matrix.backend == 'elasticsearch7' && 'elasticsearch' || matrix.backend == 'elasticsearch8' && 'elasticsearch' || 'opensearch' }} + REDIS_ENABLE: true + REDIS_HOST: localhost + REDIS_PORT: 6379 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e867050b..f550c8cb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,7 +31,8 @@ repos: ] additional_dependencies: [ "types-attrs", - "types-requests" + "types-requests", + "types-redis" ] - repo: https://github.com/PyCQA/pydocstyle rev: 6.1.1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 21151ced..7892d554 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Moved SFEOS Tools to its own repository at [Healy-Hyperspatial/sfeos-tools](https://github.com/Healy-Hyperspatial/sfeos-tools). The CLI package is now maintained separately. [#PR_NUMBER] - CloudFerro logo to sponsors and supporters list [#485](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/485) - Latest news section to README [#485](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/485) +- Added Redis caching configuration for navigation pagination support, enabling proper `prev` and `next` links in paginated responses. [#488](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/488) ### Changed diff --git a/Makefile b/Makefile index 204b31a1..34b13815 100644 --- a/Makefile +++ b/Makefile @@ -117,4 +117,4 @@ docs-image: .PHONY: docs docs: docs-image docker compose -f compose.docs.yml \ - run docs \ No newline at end of file + run docs diff --git a/README.md b/README.md index f23300ae..0981c548 100644 --- a/README.md +++ b/README.md @@ -341,6 +341,31 @@ You can customize additional settings in your `.env` file: > [!NOTE] > The variables `ES_HOST`, `ES_PORT`, `ES_USE_SSL`, `ES_VERIFY_CERTS` and `ES_TIMEOUT` apply to both Elasticsearch and OpenSearch backends, so there is no need to rename the key names to `OS_` even if you're using OpenSearch. +**Redis for Navigation:** +These Redis configuration variables enable proper navigation functionality in STAC FastAPI. The Redis cache stores navigation state for paginated results, allowing the system to maintain previous page links using tokens. The configuration supports either Redis Sentinel or standalone Redis setups. + +| Variable | Description | Default | Required | +|-------------------------------|----------------------------------------------------------------------------------------------|--------------------------|---------------------------------------------------------------------------------------------| +| **General** | | | | +| `REDIS_ENABLE` | Enables or disables Redis caching for navigation. Set to `true` to use Redis, or `false` to disable. | `false` | **Required** (determines whether Redis is used at all) | +| **Redis Sentinel** | | | | +| `REDIS_SENTINEL_HOSTS` | Comma-separated list of Redis Sentinel hostnames/IP addresses. | `""` | Conditional (required if using Sentinel) | +| `REDIS_SENTINEL_PORTS` | Comma-separated list of Redis Sentinel ports (must match order). | `"26379"` | Conditional (required if using Sentinel) | +| `REDIS_SENTINEL_MASTER_NAME` | Name of the Redis master node in Sentinel configuration. | `"master"` | Conditional (required if using Sentinel) | +| **Redis** | | | | +| `REDIS_HOST` | Redis server hostname or IP address for Redis configuration. | `""` | Conditional (required for standalone Redis) | +| `REDIS_PORT` | Redis server port for Redis configuration. | `6379` | Conditional (required for standalone Redis) | +| **Both** | | | | +| `REDIS_DB` | Redis database number to use for caching. | `0` (Sentinel) / `0` (Standalone) | Optional | +| `REDIS_MAX_CONNECTIONS` | Maximum number of connections in the Redis connection pool. | `10` | Optional | +| `REDIS_RETRY_TIMEOUT` | Enable retry on timeout for Redis operations. | `true` | Optional | +| `REDIS_DECODE_RESPONSES` | Automatically decode Redis responses to strings. | `true` | Optional | +| `REDIS_CLIENT_NAME` | Client name identifier for Redis connections. | `"stac-fastapi-app"` | Optional | +| `REDIS_HEALTH_CHECK_INTERVAL` | Interval in seconds for Redis health checks. | `30` | Optional | + +> [!NOTE] +> Use either the Sentinel configuration (`REDIS_SENTINEL_HOSTS`, `REDIS_SENTINEL_PORTS`, `REDIS_SENTINEL_MASTER_NAME`) OR the Redis configuration (`REDIS_HOST`, `REDIS_PORT`), but not both. + ## Datetime-Based Index Management ### Overview diff --git a/compose.yml b/compose.yml index 8c83ae12..82cf9fca 100644 --- a/compose.yml +++ b/compose.yml @@ -23,6 +23,9 @@ services: - BACKEND=elasticsearch - DATABASE_REFRESH=true - ENABLE_COLLECTIONS_SEARCH_ROUTE=true + - REDIS_ENABLE=true + - REDIS_HOST=redis + - REDIS_PORT=6379 ports: - "8080:8080" volumes: @@ -31,6 +34,7 @@ services: - ./esdata:/usr/share/elasticsearch/data depends_on: - elasticsearch + - redis command: bash -c "./scripts/wait-for-it-es.sh es-container:9200 && python -m stac_fastapi.elasticsearch.app" @@ -58,6 +62,9 @@ services: - BACKEND=opensearch - STAC_FASTAPI_RATE_LIMIT=200/minute - ENABLE_COLLECTIONS_SEARCH_ROUTE=true + - REDIS_ENABLE=true + - REDIS_HOST=redis + - REDIS_PORT=6379 ports: - "8082:8082" volumes: @@ -66,6 +73,7 @@ services: - ./osdata:/usr/share/opensearch/data depends_on: - opensearch + - redis command: bash -c "./scripts/wait-for-it-es.sh os-container:9202 && python -m stac_fastapi.opensearch.app" @@ -96,3 +104,14 @@ services: - ./opensearch/snapshots:/usr/share/opensearch/snapshots ports: - "9202:9202" + + redis: + image: redis:7-alpine + hostname: redis + ports: + - "6379:6379" + volumes: + - redis_test_data:/data + command: redis-server +volumes: + redis_test_data: diff --git a/stac_fastapi/core/setup.py b/stac_fastapi/core/setup.py index 92442997..8bd2c495 100644 --- a/stac_fastapi/core/setup.py +++ b/stac_fastapi/core/setup.py @@ -20,6 +20,7 @@ "jsonschema~=4.0.0", "slowapi~=0.1.9", ] +extra_reqs = {"redis": ["redis~=6.4.0"]} setup( name="stac_fastapi_core", @@ -43,4 +44,5 @@ packages=find_namespace_packages(), zip_safe=False, install_requires=install_requires, + extras_require=extra_reqs, ) diff --git a/stac_fastapi/core/stac_fastapi/core/core.py b/stac_fastapi/core/stac_fastapi/core/core.py index f4a9058e..ceb17bd2 100644 --- a/stac_fastapi/core/stac_fastapi/core/core.py +++ b/stac_fastapi/core/stac_fastapi/core/core.py @@ -24,9 +24,10 @@ from stac_fastapi.core.base_settings import ApiBaseSettings from stac_fastapi.core.datetime_utils import format_datetime_range from stac_fastapi.core.models.links import PagingLinks +from stac_fastapi.core.redis_utils import redis_pagination_links from stac_fastapi.core.serializers import CollectionSerializer, ItemSerializer from stac_fastapi.core.session import Session -from stac_fastapi.core.utilities import filter_fields +from stac_fastapi.core.utilities import filter_fields, get_bool_env from stac_fastapi.extensions.core.transaction import AsyncBaseTransactionsClient from stac_fastapi.extensions.core.transaction.request import ( PartialCollection, @@ -273,6 +274,7 @@ async def all_collections( A Collections object containing all the collections in the database and links to various resources. """ base_url = str(request.base_url) + redis_enable = get_bool_env("REDIS_ENABLE", default=False) global_max_limit = ( int(os.getenv("STAC_GLOBAL_COLLECTION_MAX_LIMIT")) @@ -428,6 +430,14 @@ async def all_collections( }, ] + if redis_enable: + await redis_pagination_links( + current_url=str(request.url), + token=token, + next_token=next_token, + links=links, + ) + if next_token: next_link = PagingLinks(next=next_token, request=request).link_next() links.append(next_link) @@ -772,8 +782,8 @@ async def post_search( search_request.limit = limit base_url = str(request.base_url) - search = self.database.make_search() + redis_enable = get_bool_env("REDIS_ENABLE", default=False) if search_request.ids: search = self.database.apply_ids_filter( @@ -877,6 +887,33 @@ async def post_search( ] links = await PagingLinks(request=request, next=next_token).get_links() + collection_links = [] + if search_request.collections: + for collection_id in search_request.collections: + collection_links.extend( + [ + { + "rel": "collection", + "type": "application/json", + "href": urljoin(base_url, f"collections/{collection_id}"), + }, + { + "rel": "parent", + "type": "application/json", + "href": urljoin(base_url, f"collections/{collection_id}"), + }, + ] + ) + links.extend(collection_links) + + if redis_enable: + await redis_pagination_links( + current_url=str(request.url), + token=token_param, + next_token=next_token, + links=links, + ) + return stac_types.ItemCollection( type="FeatureCollection", features=items, diff --git a/stac_fastapi/core/stac_fastapi/core/redis_utils.py b/stac_fastapi/core/stac_fastapi/core/redis_utils.py new file mode 100644 index 00000000..ad2b6837 --- /dev/null +++ b/stac_fastapi/core/stac_fastapi/core/redis_utils.py @@ -0,0 +1,179 @@ +"""Utilities for connecting to and managing Redis connections.""" + +import json +import logging +from typing import List, Optional, Tuple + +from pydantic_settings import BaseSettings +from redis import asyncio as aioredis +from redis.asyncio.sentinel import Sentinel + +logger = logging.getLogger(__name__) + + +class RedisSentinelSettings(BaseSettings): + """Configuration settings for connecting to Redis Sentinel.""" + + REDIS_SENTINEL_HOSTS: str = "" + REDIS_SENTINEL_PORTS: str = "26379" + REDIS_SENTINEL_MASTER_NAME: str = "master" + REDIS_DB: int = 0 + + REDIS_MAX_CONNECTIONS: int = 10 + REDIS_DECODE_RESPONSES: bool = True + REDIS_CLIENT_NAME: str = "stac-fastapi-app" + REDIS_HEALTH_CHECK_INTERVAL: int = 30 + REDIS_SELF_LINK_TTL: int = 1800 + + def get_sentinel_nodes(self) -> List[Tuple[str, int]]: + """Return list of (host, port) tuples.""" + try: + hosts = json.loads(self.REDIS_SENTINEL_HOSTS) + ports = json.loads(self.REDIS_SENTINEL_PORTS) + except json.JSONDecodeError: + hosts = [ + h.strip() for h in self.REDIS_SENTINEL_HOSTS.split(",") if h.strip() + ] + ports = [ + int(p.strip()) + for p in self.REDIS_SENTINEL_PORTS.split(",") + if p.strip() + ] + + if len(ports) == 1 and len(hosts) > 1: + ports = ports * len(hosts) + + return list(zip(hosts, ports)) + + +class RedisSettings(BaseSettings): + """Configuration settings for connecting to a standalone Redis instance.""" + + REDIS_HOST: str = "" + REDIS_PORT: int = 6379 + REDIS_DB: int = 0 + + REDIS_MAX_CONNECTIONS: int = 10 + REDIS_DECODE_RESPONSES: bool = True + REDIS_CLIENT_NAME: str = "stac-fastapi-app" + REDIS_HEALTH_CHECK_INTERVAL: int = 30 + REDIS_SELF_LINK_TTL: int = 1800 + + +sentinel_settings = RedisSentinelSettings() +standalone_settings = RedisSettings() + +redis: Optional[aioredis.Redis] = None + + +async def connect_redis() -> Optional[aioredis.Redis]: + """Initialize global Redis connection (Sentinel or Standalone).""" + global redis + if redis: + return redis + + try: + # Prefer Sentinel if configured + if sentinel_settings.REDIS_SENTINEL_HOSTS.strip(): + sentinel_nodes = sentinel_settings.get_sentinel_nodes() + sentinel = Sentinel( + sentinel_nodes, + decode_responses=True, + ) + redis = sentinel.master_for( + service_name=sentinel_settings.REDIS_SENTINEL_MASTER_NAME, + db=sentinel_settings.REDIS_DB, + decode_responses=True, + client_name=sentinel_settings.REDIS_CLIENT_NAME, + max_connections=sentinel_settings.REDIS_MAX_CONNECTIONS, + health_check_interval=sentinel_settings.REDIS_HEALTH_CHECK_INTERVAL, + ) + await redis.ping() + logger.info("✅ Connected to Redis Sentinel") + return redis + + # Fallback to standalone + if standalone_settings.REDIS_HOST.strip(): + redis = aioredis.Redis( + host=standalone_settings.REDIS_HOST, + port=standalone_settings.REDIS_PORT, + db=standalone_settings.REDIS_DB, + decode_responses=True, + client_name=standalone_settings.REDIS_CLIENT_NAME, + health_check_interval=standalone_settings.REDIS_HEALTH_CHECK_INTERVAL, + ) + await redis.ping() + logger.info("✅ Connected to standalone Redis") + return redis + + logger.warning("⚠️ No Redis configuration found — skipping connection.") + return None + + except Exception as e: + logger.error(f"❌ Failed to connect to Redis: {e}") + redis = None + return None + + +async def close_redis(): + """Close global Redis connection.""" + global redis + if redis: + await redis.close() + redis = None + logger.info("Redis connection closed.") + + +async def save_self_link( + redis: aioredis.Redis, token: Optional[str], self_href: str +) -> None: + """Save current self link for token.""" + if not token: + return + + ttl = ( + sentinel_settings.REDIS_SELF_LINK_TTL + if sentinel_settings.REDIS_SENTINEL_HOSTS.strip() + else standalone_settings.REDIS_SELF_LINK_TTL + ) + + await redis.setex(f"nav:self:{token}", ttl, self_href) + + +async def get_prev_link(redis: aioredis.Redis, token: Optional[str]) -> Optional[str]: + """Return previous page link for token.""" + if not token: + return None + return await redis.get(f"nav:self:{token}") + + +async def redis_pagination_links( + current_url: str, + token: str, + next_token: str, + links: list, + redis_conn: Optional[aioredis.Redis] = None, +) -> None: + """Manage pagination links stored in Redis.""" + redis_conn = redis_conn or await connect_redis() + if not redis_conn: + logger.warning("Redis not available for pagination.") + return + + try: + if next_token: + await save_self_link(redis_conn, next_token, current_url) + + prev_link = await get_prev_link(redis_conn, token) + if prev_link: + links.insert( + 0, + { + "rel": "prev", + "type": "application/json", + "method": "GET", + "href": prev_link, + }, + ) + except Exception as e: + logger.warning(f"Redis pagination failed: {e}") diff --git a/stac_fastapi/elasticsearch/setup.py b/stac_fastapi/elasticsearch/setup.py index 1751df78..5d95fcdb 100644 --- a/stac_fastapi/elasticsearch/setup.py +++ b/stac_fastapi/elasticsearch/setup.py @@ -21,9 +21,11 @@ "pre-commit~=3.0.0", "ciso8601~=2.3.0", "httpx>=0.24.0,<0.28.0", + "stac-fastapi-core[redis]==6.5.1", ], "docs": ["mkdocs~=1.4.0", "mkdocs-material~=9.0.0", "pdocs~=1.2.0"], "server": ["uvicorn[standard]~=0.23.0"], + "redis": ["stac-fastapi-core[redis]==6.5.1"], } setup( diff --git a/stac_fastapi/opensearch/setup.py b/stac_fastapi/opensearch/setup.py index d7727267..2522c581 100644 --- a/stac_fastapi/opensearch/setup.py +++ b/stac_fastapi/opensearch/setup.py @@ -22,9 +22,11 @@ "pre-commit~=3.0.0", "ciso8601~=2.3.0", "httpx>=0.24.0,<0.28.0", + "stac-fastapi-core[redis]==6.5.1", ], "docs": ["mkdocs~=1.4.0", "mkdocs-material~=9.0.0", "pdocs~=1.2.0"], "server": ["uvicorn[standard]~=0.23.0"], + "redis": ["stac-fastapi-core[redis]==6.5.1"], } setup( diff --git a/stac_fastapi/tests/redis/__init__.py b/stac_fastapi/tests/redis/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/stac_fastapi/tests/redis/test_redis_pagination.py b/stac_fastapi/tests/redis/test_redis_pagination.py new file mode 100644 index 00000000..2f4c93e1 --- /dev/null +++ b/stac_fastapi/tests/redis/test_redis_pagination.py @@ -0,0 +1,78 @@ +import uuid + +import pytest + +from ..conftest import create_collection, create_item + + +@pytest.mark.asyncio +async def test_search_pagination_uses_redis_cache( + app_client, txn_client, load_test_data +): + """Test Redis caching and navigation for the /search endpoint.""" + + collection = load_test_data("test_collection.json") + collection_id = f"test-pagination-collection-{uuid.uuid4()}" + collection["id"] = collection_id + await create_collection(txn_client, collection) + + for i in range(5): + item = load_test_data("test_item.json") + item["id"] = f"test-pagination-item-{uuid.uuid4()}" + item["collection"] = collection_id + await create_item(txn_client, item) + + resp = await app_client.post( + "/search", json={"collections": [collection_id], "limit": 1} + ) + resp_json = resp.json() + + next_link = next( + (link for link in resp_json["links"] if link["rel"] == "next"), None + ) + next_token = next_link["body"]["token"] + + resp2 = await app_client.post( + "/search", + json={"collections": [collection_id], "limit": 1, "token": next_token}, + ) + resp2_json = resp2.json() + + prev_link = next( + (link for link in resp2_json["links"] if link["rel"] == "prev"), None + ) + assert prev_link is not None + + +@pytest.mark.asyncio +async def test_collections_pagination_uses_redis_cache( + app_client, txn_client, load_test_data +): + """Test Redis caching and navigation for the /collection endpoint.""" + + collection_data = load_test_data("test_collection.json") + for i in range(5): + collection = collection_data.copy() + collection["id"] = f"test-collection-pagination-{uuid.uuid4()}" + collection["title"] = f"Test Collection Pagination {i}" + await create_collection(txn_client, collection) + + resp = await app_client.get("/collections", params={"limit": 1}) + assert resp.status_code == 200 + resp1_json = resp.json() + + next_link = next( + (link for link in resp1_json["links"] if link["rel"] == "next"), None + ) + next_token = next_link["href"].split("token=")[1] + + resp2 = await app_client.get( + "/collections", params={"limit": 1, "token": next_token} + ) + assert resp2.status_code == 200 + resp2_json = resp2.json() + + prev_link = next( + (link for link in resp2_json["links"] if link["rel"] == "prev"), None + ) + assert prev_link is not None diff --git a/stac_fastapi/tests/redis/test_redis_utils.py b/stac_fastapi/tests/redis/test_redis_utils.py new file mode 100644 index 00000000..a7dc8338 --- /dev/null +++ b/stac_fastapi/tests/redis/test_redis_utils.py @@ -0,0 +1,44 @@ +import pytest + +from stac_fastapi.core.redis_utils import connect_redis, get_prev_link, save_self_link + + +@pytest.mark.asyncio +async def test_redis_connection(): + """Test Redis connection.""" + redis = await connect_redis() + + if redis is None: + pytest.skip("Redis not configured") + + await redis.set("string_key", "string_value") + string_value = await redis.get("string_key") + assert string_value == "string_value" + + exists = await redis.exists("string_key") + assert exists == 1 + + await redis.delete("string_key") + deleted_value = await redis.get("string_key") + assert deleted_value is None + + +@pytest.mark.asyncio +async def test_redis_utils_functions(): + redis = await connect_redis() + if redis is None: + pytest.skip("Redis not configured") + + token = "test_token_123" + self_link = "http://mywebsite.com/search?token=test_token_123" + + await save_self_link(redis, token, self_link) + retrieved_link = await get_prev_link(redis, token) + assert retrieved_link == self_link + + await save_self_link(redis, None, "should_not_save") + null_result = await get_prev_link(redis, None) + assert null_result is None + + non_existent = await get_prev_link(redis, "non_existent_token") + assert non_existent is None