Import Redis cache backend from defunkt kinto-redis

.github/workflows/test.yml

@@ -66,6 +66,15 @@ jobs:
           --health-retries 5
         ports:
           - 11211:11211
+      redis:
+        image: redis
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 6379:6379
 
     steps:
       - uses: actions/checkout@v6

Dockerfile

@@ -17,7 +17,7 @@ COPY constraints.txt pyproject.toml ./
 RUN pip install --upgrade pip && pip install -r constraints.txt
 COPY kinto/ kinto/
 RUN cp -r /kinto-admin/kinto/plugins/admin/build kinto/plugins/admin/
-RUN pip install ".[postgresql,memcached,monitoring]" -c constraints.txt && pip install kinto-attachment kinto-emailer httpie
+RUN pip install ".[postgresql,memcached,redis,monitoring]" -c constraints.txt && pip install kinto-attachment kinto-emailer httpie
 
 FROM python:3.10-slim-bullseye
 RUN groupadd --gid 10001 app && \

Makefile

@@ -42,9 +42,12 @@ install-postgres: $(INSTALL_STAMP) $(DEV_STAMP) ## install postgresql support
 install-memcached: $(INSTALL_STAMP) $(DEV_STAMP) ## install memcached support
 	$(VENV)/bin/pip install -Ue ".[memcached]" -c constraints.txt
 
+install-redis: $(INSTALL_STAMP) $(DEV_STAMP) ## install redis support
+	$(VENV)/bin/pip install -Ue ".[redis]" -c constraints.txt
+
 install-dev: $(INSTALL_STAMP) $(DEV_STAMP) ## install dependencies and everything needed to run tests
 $(DEV_STAMP): $(PYTHON) constraints.txt
-	$(VENV)/bin/pip install -Ue ".[dev,test,monitoring,postgresql,memcached]" -c constraints.txt
+	$(VENV)/bin/pip install -Ue ".[dev,test,monitoring,postgresql,memcached,redis]" -c constraints.txt
 	touch $(DEV_STAMP)
 
 install-docs: $(DOC_STAMP) ## install dependencies to build the docs
@@ -87,8 +90,10 @@ tests-raw: version-file install-dev
 .PHONY: test-deps
 test-deps:
 	docker pull memcached
+	docker pull redis
 	docker pull postgres
 	docker run -p 11211:11211 --name kinto-memcached -d memcached || echo "cannot start memcached, already exists?"
+	docker run -p 6379:6379 --name kinto-redis -d redis || echo "cannot start redis, already exists?"
 	docker run -p 5432:5432 --name kinto-postgres -e POSTGRES_PASSWORD=postgres -d postgres  || echo "cannot start postgres, already exists?"
 	sleep 2
 	PGPASSWORD=postgres psql -c "CREATE DATABASE testdb ENCODING 'UTF8' TEMPLATE template0;" -U postgres -h localhost
@@ -131,6 +136,7 @@ clean: ## remove built files and start fresh
 	rm -fr kinto/plugins/admin/build/ kinto/plugins/admin/node_modules/
 	docker rm -f kinto-memcached || echo ""
 	docker rm -f kinto-postgres || echo ""
+	docker rm -f kinto-redis || echo ""
 
 docs: install-docs ## build the docs
 	$(VENV)/bin/sphinx-build -a -W -n -b html -d $(SPHINX_BUILDDIR)/doctrees docs $(SPHINX_BUILDDIR)/html

constraints.in

@@ -16,6 +16,8 @@ python-rapidjson
 # optional dependencies
 # memcached
 python-memcached
+# redis
+redis
 # postgresql
 SQLAlchemy < 3
 psycopg2-binary
@@ -36,3 +38,4 @@ webtest
 # dev
 build
 ruff
+setuptools<82.0

constraints.txt

@@ -1,8 +1,8 @@
 #
-# This file is autogenerated by pip-compile with Python 3.12
+# This file is autogenerated by pip-compile with Python 3.14
 # by the following command:
 #
-#    pip-compile --output-file=constraints.txt --strip-extras constraints.in
+#    pip-compile --allow-unsafe --output-file=constraints.txt --strip-extras constraints.in
 #
 arrow==1.3.0
     # via isoduration
@@ -37,9 +37,7 @@ execnet==2.0.2
 fqdn==1.5.1
     # via jsonschema
 greenlet==3.1.1
-    # via
-    #   playwright
-    #   sqlalchemy
+    # via playwright
 hupper==1.12
     # via pyramid
 idna==3.7
@@ -139,6 +137,8 @@ pyyaml==6.0.1
     # via
     #   bravado-core
     #   swagger-spec-validator
+redis==7.1.0
+    # via -r constraints.in
 referencing==0.32.1
     # via
     #   jsonschema
@@ -228,4 +228,9 @@ zope-sqlalchemy==4.1
     # via -r constraints.in
 
 # The following packages are considered to be unsafe in a requirements file:
-# setuptools
+setuptools==81.0.0
+    # via
+    #   -r constraints.in
+    #   pyramid
+    #   zope-deprecation
+    #   zope-interface

docs/configuration/settings.rst

@@ -190,6 +190,16 @@ You would need to install the memcached dependencies: ``pip install kinto[memcac
     kinto.cache_backend = kinto.core.cache.memcached
     kinto.cache_hosts = 127.0.0.1:11211 127.0.0.2:11211
 
+For **Redis**
+
+You would need to install the Redis dependencies: ``pip install kinto[redis]``
+
+.. code-block:: ini
+
+    kinto.cache_backend = kinto.core.cache.redis
+    # kinto.cache_url = redis://127.0.0.1:6379/0
+
+
 Permissions
 :::::::::::
 

docs/core/cache.rst

@@ -27,6 +27,10 @@ Memcached
 
 .. autoclass:: kinto.core.cache.memcached.Cache
 
+Redis
+=====
+
+.. autoclass:: kinto.core.cache.redis.Cache
 
 API
 ===

docs/requirements.txt

@@ -9,3 +9,4 @@ webtest==3.0.7
 pyramid==2.0.2
 python-rapidjson==1.23
 SQLAlchemy==2.0.46
+setuptools<82.0

kinto/__main__.py

@@ -83,7 +83,7 @@ def main(args=None):
             )
             subparser.add_argument(
                 "--cache-backend",
-                help="{memory,postgresql,memcached}",
+                help="{memory,postgresql,memcached,redis}",
                 dest="cache-backend",
                 required=False,
                 default=None,
@@ -176,13 +176,14 @@ def main(args=None):
             while True:
                 prompt = (
                     "Select the cache backend you would like to use: "
-                    "(1 - postgresql, 2 - memcached, default - memory) "
+                    "(1 - postgresql, 2 - memcached, 3 - redis, default - memory) "
                 )
                 answer = input(prompt).strip()
                 try:
                     cache_backends = {
                         "1": "postgresql",
                         "2": "memcached",
+                        "3": "redis",
                         "": "memory",
                     }
                     cache_backend = cache_backends[answer]
@@ -205,6 +206,11 @@ def main(args=None):
                 import memcache  # NOQA
             except ImportError:
                 subprocess.check_call([sys.executable, "-m", "pip", "install", "kinto[memcached]"])
+        elif cache_backend == "redis":
+            try:
+                import redis  # NOQA
+            except ImportError:
+                subprocess.check_call([sys.executable, "-m", "pip", "install", "kinto[redis]"])
 
     elif which_command == "migrate":
         dry_run = parsed_args["dry_run"]

kinto/config/__init__.py

@@ -54,6 +54,7 @@ def render_template(template, destination, **kwargs):
         "cache_backend": "kinto.core.cache.memcached",
         "cache_url": "127.0.0.1:11211 127.0.0.2:11211",
     },
+    "redis": {"cache_backend": "kinto.core.cache.redis", "cache_url": "redis://127.0.0.1/0"},
     "memory": {"cache_backend": "kinto.core.cache.memory", "cache_url": ""},
 }
 

kinto/config/kinto.tpl

@@ -65,6 +65,9 @@ kinto.cache_url = {cache_url}
 # kinto.cache_backend = kinto.core.cache.memcached
 # kinto.cache_hosts = 127.0.0.1:11211
 
+# kinto.cache_backend = kinto.core.cache.redis
+# kinto.cache_url = redis://127.0.0.1:6379/0
+
 # Permissions.
 # https://kinto.readthedocs.io/en/latest/configuration/settings.html#permissions
 #

kinto/core/cache/redis.py

@@ -0,0 +1,124 @@
+import logging
+from functools import wraps
+from urllib.parse import urlparse
+
+from kinto.core.cache import CacheBase
+from kinto.core.storage import exceptions
+from kinto.core.utils import json, redis
+
+
+logger = logging.getLogger(__name__)
+
+
+def wrap_redis_error(func):
+    @wraps(func)
+    def wrapped(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except redis.exceptions.RedisError as e:
+            logger.exception(e)
+            raise exceptions.BackendError(original=e)
+
+    return wrapped
+
+
+class Cache(CacheBase):
+    """Cache backend implementation using Redis.
+
+    Enable in configuration::
+
+        kinto.cache_backend = kinto.core.cache.memcached
+
+    *(Optional)* Instance location URI can be customized::
+
+        kinto.cache_url = redis://localhost:6379/1
+
+    A threaded connection pool is enabled by default::
+
+        kinto.cache_pool_size = 50
+        kinto.cache_pool_timeout = 30
+
+    If the database is used for multiple Kinto deployement cache, you
+    may want to add a prefix to every key to avoid collision::
+
+        kinto.cache_prefix = stack1_
+
+    :noindex:
+
+    """
+
+    def __init__(self, client, *args, **kwargs):
+        super(Cache, self).__init__(*args, **kwargs)
+        self._client = client
+
+    @property
+    def settings(self):
+        return dict(self._client.connection_pool.connection_kwargs)
+
+    def initialize_schema(self, dry_run=False):
+        # Nothing to do.
+        pass
+
+    @wrap_redis_error
+    def flush(self):
+        self._client.flushdb()
+
+    @wrap_redis_error
+    def ttl(self, key):
+        return self._client.ttl(self.prefix + key)
+
+    @wrap_redis_error
+    def expire(self, key, ttl):
+        self._client.pexpire(self.prefix + key, int(ttl * 1000))
+
+    @wrap_redis_error
+    def set(self, key, value, ttl):
+        if isinstance(value, bytes):
+            raise TypeError("a string-like object is required, not 'bytes'")
+        value = json.dumps(value)
+        self._client.psetex(self.prefix + key, int(ttl * 1000), value)
+
+    @wrap_redis_error
+    def get(self, key):
+        value = self._client.get(self.prefix + key)
+        if value:
+            self.metrics_backend.count_hit()
+            value = value.decode("utf-8")
+            return json.loads(value)
+        self.metrics_backend.count_miss()
+
+    @wrap_redis_error
+    def delete(self, key):
+        value = self.get(key)
+        self._client.delete(self.prefix + key)
+        return value
+
+
+def create_from_config(config, prefix=""):
+    """Redis client instantiation from settings."""
+    settings = config.get_settings()
+    uri = settings[prefix + "url"]
+    uri = urlparse(uri)
+    kwargs = {
+        "host": uri.hostname or "localhost",
+        "port": uri.port or 6379,
+        "password": uri.password or None,
+        "db": int(uri.path[1:]) if uri.path else 0,
+    }
+
+    pool_size = settings.get(prefix + "pool_size")
+    if pool_size is not None:
+        kwargs["max_connections"] = int(pool_size)
+
+    block_timeout = settings.get(prefix + "pool_timeout")
+    if block_timeout is not None:
+        kwargs["timeout"] = float(block_timeout)
+
+    connection_pool = redis.BlockingConnectionPool(**kwargs)
+    return redis.StrictRedis(connection_pool=connection_pool)
+
+
+def load_from_config(config):
+    settings = config.get_settings()
+    client = create_from_config(config, prefix="cache_")
+    return Cache(client, cache_prefix=settings["cache_prefix"])

kinto/core/testing.py

@@ -10,13 +10,14 @@
 from kinto.core import DEFAULT_SETTINGS
 from kinto.core.cornice import errors as cornice_errors
 from kinto.core.storage import generators
-from kinto.core.utils import encode64, follow_subrequest, memcache, sqlalchemy
+from kinto.core.utils import encode64, follow_subrequest, memcache, redis, sqlalchemy
 from kinto.plugins import prometheus, statsd
 
 
 skip_if_ci = unittest.skipIf("CI" in os.environ, "ci")
 skip_if_no_postgresql = unittest.skipIf(sqlalchemy is None, "postgresql is not installed.")
 skip_if_no_memcached = unittest.skipIf(memcache is None, "memcached is not installed.")
+skip_if_no_redis = unittest.skipIf(redis is None, "redis is not installed.")
 skip_if_no_statsd = unittest.skipIf(not statsd.statsd_module, "statsd is not installed.")
 skip_if_no_prometheus = unittest.skipIf(
     not prometheus.prometheus_module, "prometheus is not installed."

kinto/core/utils.py

@@ -33,6 +33,11 @@
 except ImportError:  # pragma: no cover
     memcache = None
 
+try:
+    import redis
+except ImportError:  # pragma: no cover
+    redis = None
+
 
 class json:
     def dumps(v, **kw):

pyproject.toml

@@ -62,12 +62,13 @@ requires = ["setuptools>=64", "setuptools_scm>=8"]
 build-backend = "setuptools.build_meta"
 
 [project.optional-dependencies]
-redis = [
-    "kinto_redis",
-]
 memcached = [
     "python-memcached",
 ]
+redis = [
+    "redis",
+]
+
 postgresql = [
     "SQLAlchemy < 3",
     "psycopg2-binary",