add redis

Author: Yuri Zmytrakov

.pre-commit-config.yaml

@@ -31,7 +31,8 @@ repos:
         ]
         additional_dependencies: [
           "types-attrs",
-          "types-requests"
+          "types-requests",
+          "types-redis"
         ]
   - repo: https://github.com/PyCQA/pydocstyle
     rev: 6.1.1

Makefile

@@ -63,7 +63,7 @@ docker-shell-os:
 
 .PHONY: test-elasticsearch
 test-elasticsearch:
-	-$(run_es) /bin/bash -c 'export && ./scripts/wait-for-it-es.sh elasticsearch:9200 && cd stac_fastapi/tests/ && pytest'
+	-$(run_es) /bin/bash -c 'pip install redis export && ./scripts/wait-for-it-es.sh elasticsearch:9200 && cd stac_fastapi/tests/ && pytest'
 	docker compose down
 
 .PHONY: test-opensearch

compose.yml

@@ -22,6 +22,8 @@ services:
       - ES_VERIFY_CERTS=false
       - BACKEND=elasticsearch
       - DATABASE_REFRESH=true
+      - REDIS_HOST=redis
+      - REDIS_PORT=6379
     ports:
       - "8080:8080"
     volumes:
@@ -30,6 +32,7 @@ services:
       - ./esdata:/usr/share/elasticsearch/data
     depends_on:
       - elasticsearch
+      - redis
     command:
       bash -c "./scripts/wait-for-it-es.sh es-container:9200 && python -m stac_fastapi.elasticsearch.app"
 
@@ -94,3 +97,10 @@ services:
       - ./opensearch/snapshots:/usr/share/opensearch/snapshots
     ports:
       - "9202:9202"
+
+  redis:
+    container_name: stac-redis
+    image: redis:7.2-alpine
+    restart: always
+    ports:
+      - "6379:6379"

docker-compose.redis.yml

@@ -0,0 +1,24 @@
+version: '3.8'
+
+services:
+  redis:
+    image: redis:7-alpine
+    container_name: stac-app-redis
+    ports:
+      - "6379:6379"
+    volumes:
+      - redis_data:/data
+    command: >
+      redis-server
+      --appendonly yes
+      --maxmemory 256mb
+      --maxmemory-policy allkeys-lru
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+    restart: unless-stopped
+
+volumes:
+  redis_data:

stac_fastapi/core/setup.py

@@ -19,6 +19,7 @@
     "pygeofilter~=0.3.1",
     "jsonschema~=4.0.0",
     "slowapi~=0.1.9",
+    "redis==6.4.0",
 ]
 
 setup(

stac_fastapi/core/stac_fastapi/core/core.py

@@ -24,6 +24,12 @@
 from stac_fastapi.core.base_settings import ApiBaseSettings
 from stac_fastapi.core.datetime_utils import format_datetime_range
 from stac_fastapi.core.models.links import PagingLinks
+from stac_fastapi.core.redis_utils import (
+    add_previous_link_if_exists,
+    cache_current_url,
+    cache_previous_url,
+    connect_redis,
+)
 from stac_fastapi.core.serializers import CollectionSerializer, ItemSerializer
 from stac_fastapi.core.session import Session
 from stac_fastapi.core.utilities import filter_fields
@@ -237,6 +243,13 @@ async def all_collections(self, **kwargs) -> stac_types.Collections:
         base_url = str(request.base_url)
         limit = int(request.query_params.get("limit", os.getenv("STAC_ITEM_LIMIT", 10)))
         token = request.query_params.get("token")
+        current_url = str(request.url)
+        redis = None
+        try:
+            redis = await connect_redis()
+        except Exception as e:
+            logger.error(f"Redis connection error: {e}")
+            redis = None
 
         collections, next_token = await self.database.get_all_collections(
             token=token, limit=limit, request=request
@@ -252,6 +265,12 @@ async def all_collections(self, **kwargs) -> stac_types.Collections:
             },
         ]
 
+        await add_previous_link_if_exists(
+            redis, links, "collections", current_url, token
+        )
+        if redis:
+            await cache_previous_url(redis, current_url, "collections")
+
         if next_token:
             next_link = PagingLinks(next=next_token, request=request).link_next()
             links.append(next_link)
@@ -310,20 +329,19 @@ async def item_collection(
         """
         request: Request = kwargs["request"]
         token = request.query_params.get("token")
-        if not hasattr(self, '_prev_links'):
-            self._prev_links = {}
-        
-        session_id = request.cookies.get('stac_session', 'default_session')
-        current_self_link = str(request.url)
-
-        if session_id not in self._prev_links:
-            self._prev_links[session_id] = []
-        
-        history = self._prev_links[session_id]
-        if not history or current_self_link != history[-1]:
-            history.append(current_self_link)
         base_url = str(request.base_url)
 
+        current_url = str(request.url)
+
+        try:
+            redis = await connect_redis()
+        except Exception as e:
+            logger.error(f"Redis connection error: {e}")
+            redis = None
+
+        if redis:
+            await cache_current_url(redis, current_url, collection_id)
+
         collection = await self.get_collection(
             collection_id=collection_id, request=request
         )
@@ -374,21 +392,22 @@ async def item_collection(
                 "href": urljoin(str(request.base_url), f"collections/{collection_id}"),
             },
             {
-                "rel": "parent", 
+                "rel": "parent",
                 "type": "application/json",
                 "href": urljoin(str(request.base_url), f"collections/{collection_id}"),
-            }
+            },
         ]
 
         paging_links = await PagingLinks(request=request, next=next_token).get_links()
-        history = self._prev_links.get(session_id, [])
-        if len(history) > 1:
-            previous_self_link = history[-2]
-            paging_links.append({
-                "rel": "previous",
-                "type": "application/json",
-                "href": previous_self_link,
-            })        
+
+        if redis:
+            await add_previous_link_if_exists(
+                redis, paging_links, collection_id, current_url, token
+            )
+
+        if redis:
+            await cache_previous_url(redis, current_url, collection_id)
+
         links = collection_links + paging_links
 
         return stac_types.ItemCollection(
@@ -529,7 +548,15 @@ async def post_search(
             HTTPException: If there is an error with the cql2_json filter.
         """
         base_url = str(request.base_url)
+        current_url = str(request.url)
+        try:
+            redis = await connect_redis()
+        except Exception as e:
+            logger.error(f"Redis connection error: {e}")
+            redis = None
 
+        if redis:
+            await cache_current_url(redis, current_url, "search_result")
         search = self.database.make_search()
 
         if search_request.ids:
@@ -628,6 +655,14 @@ async def post_search(
         ]
         links = await PagingLinks(request=request, next=next_token).get_links()
 
+        if redis:
+            await add_previous_link_if_exists(
+                redis, links, "search_result", current_url, search_request.token
+            )
+
+        if redis:
+            await cache_previous_url(redis, current_url, "search_result")
+
         return stac_types.ItemCollection(
             type="FeatureCollection",
             features=items,

stac_fastapi/core/stac_fastapi/core/redis_utils.py

@@ -0,0 +1,161 @@
+"""Utilities for connecting to and managing Redis connections."""
+
+import logging
+import os
+from typing import Dict, List, Optional
+
+from pydantic_settings import BaseSettings
+from redis import asyncio as aioredis
+from stac_pydantic.shared import MimeTypes
+
+from stac_fastapi.core.utilities import get_bool_env
+
+redis_pool = None
+
+logger = logging.getLogger(__name__)
+
+
+class RedisSettings(BaseSettings):
+    """Configuration settings for connecting to a Redis server."""
+
+    sentinel_hosts: List[str] = os.getenv("REDIS_SENTINEL_HOSTS", "").split(",")
+    sentinel_ports: List[int] = [
+        int(port)
+        for port in os.getenv("REDIS_SENTINEL_PORTS", "").split(",")
+        if port.strip()
+    ]
+    sentinel_master_name: str = os.getenv("REDIS_SENTINEL_MASTER_NAME", "")
+    redis_db: int = int(os.getenv("REDIS_DB", "0"))
+
+    max_connections: int = int(os.getenv("REDIS_MAX_CONNECTIONS", "5"))
+    retry_on_timeout: bool = get_bool_env("REDIS_RETRY_TIMEOUT", True)
+    decode_responses: bool = get_bool_env("REDIS_DECODE_RESPONSES", True)
+    client_name: str = os.getenv("REDIS_CLIENT_NAME", "stac-fastapi-app")
+    health_check_interval: int = int(os.getenv("REDIS_HEALTH_CHECK_INTERVAL", "30"))
+
+
+redis_settings = RedisSettings()
+
+
+async def connect_redis(
+    settings: Optional[RedisSettings] = None,
+) -> Optional[aioredis.Redis]:
+    """Return a Redis connection, returning None if not configured or connection fails."""
+    global redis_pool
+    settings = settings or redis_settings
+
+    if (
+        not settings.sentinel_hosts
+        or not settings.sentinel_hosts[0]
+        or not settings.sentinel_master_name
+    ):
+        logger.warning("Redis not configured - skipping Redis operations")
+        return None
+
+    if redis_pool is None:
+        try:
+            # Create async Sentinel connection
+            sentinel = aioredis.Sentinel(
+                [
+                    (host, port)
+                    for host, port in zip(
+                        settings.sentinel_hosts, settings.sentinel_ports
+                    )
+                ],
+                decode_responses=settings.decode_responses,
+                retry_on_timeout=settings.retry_on_timeout,
+                client_name=f"{settings.client_name}-sentinel",
+            )
+
+            # Get async master connection
+            master = sentinel.master_for(
+                settings.sentinel_master_name,
+                db=settings.redis_db,
+                decode_responses=settings.decode_responses,
+                retry_on_timeout=settings.retry_on_timeout,
+                client_name=settings.client_name,
+            )
+
+            # Test the connection
+            await master.ping()
+            logger.info("Redis Sentinel connection successful!")
+
+            redis_pool = master
+
+        except Exception as e:
+            logger.error(f"Failed to connect to Redis Sentinel: {e}")
+            return None
+
+    return redis_pool
+
+
+async def close_redis() -> None:
+    """Close the Redis connection pool if it exists."""
+    global redis_pool
+    if redis_pool:
+        await redis_pool.close()
+        redis_pool = None
+
+
+async def cache_current_url(redis, current_url: str, key_suffix: str) -> None:
+    """Cache the current URL for pagination."""
+    if not redis:
+        return
+
+    try:
+        current_key = f"current:{key_suffix}"
+        await redis.setex(current_key, 600, current_url)
+    except Exception as e:
+        logger.error(f"Redis cache error for {key_suffix}: {e}")
+
+
+async def get_previous_url(redis, key_suffix: str, current_url: str) -> Optional[str]:
+    """Get previous URL from cache if it exists."""
+    if redis is None:
+        return None
+
+    try:
+        prev_key = f"prev:{key_suffix}"
+        previous_url = await redis.get(prev_key)
+        # REMOVE the current_url comparison - just return the cached value
+        if previous_url:
+            return previous_url
+    except Exception as e:
+        logger.error(f"Redis get previous error for {key_suffix}: {e}")
+
+    return None
+
+
+async def cache_previous_url(redis, current_url: str, key_suffix: str) -> None:
+    """Cache the current URL as previous for next request."""
+    if not redis:
+        return
+
+    try:
+        prev_key = f"prev:{key_suffix}"
+        await redis.setex(prev_key, 600, current_url)
+        print(f"DEBUG: Cached {current_url} as previous for {key_suffix}")  # Add debug
+    except Exception as e:
+        logger.error(f"Redis cache previous error for {key_suffix}: {e}")
+
+
+async def add_previous_link_if_exists(
+    redis,
+    links: List[Dict],
+    key_suffix: str,
+    current_url: str,
+    token: Optional[str] = None,
+) -> None:
+    """Add previous link to links list if it exists in cache and conditions are met."""
+    if not redis or not token:
+        return
+
+    previous_url = await get_previous_url(redis, key_suffix, current_url)
+    if previous_url:
+        links.append(
+            {
+                "rel": "previous",
+                "type": MimeTypes.json,
+                "href": previous_url,
+            }
+        )

stac_fastapi/tests/conftest.py

@@ -6,6 +6,7 @@
 
 import pytest
 import pytest_asyncio
+import redis  # noqa: F401
 from fastapi import Depends, HTTPException, security, status
 from httpx import ASGITransport, AsyncClient
 from pydantic import ConfigDict