Support Redis & Memcached cache servers

* Sign pickles before caching to prevent tampering
* Move view tracking to a receiver so it doesn't get cached

Author: ObserverOfTime

.dockerignore

@@ -63,6 +63,10 @@ coverage.xml
 status_auth.txt
 !docker/uwsgi.ini
 
+# Redis stuff
+*.rdb
+*.aof
+
 # MangAdventure stuff
 media/
 static/admin/

.env.example

@@ -78,6 +78,18 @@ DB_URL="sqlite:///db.sqlite3"
 ## Example (TLS): smtp+tls://admin:6ceab2590b52c249741b@example.com:587
 EMAIL_URL="smtp+tls://<user>:<password>@smtp.gmail.com"
 
+# The URL of your cache server. Special characters must be urlencoded.
+# You can set multiple servers by separating them with a semicolon.
+## Redis:
+### Example (basic): redis://127.0.0.1:6379
+### Example (authenticated): redis://username:password@127.0.0.1:6379
+### Example (replicas): redis://127.0.0.1:6379;redis://127.0.0.1:6378
+## Memcached:
+### Example (HTTP): 127.0.0.1:11211
+### Example (socket): unix:/var/tmp/memcached.sock
+### Example (distributed): 172.19.26.240:11211;172.19.26.242:11211
+CACHE_URL=""
+
 # The default e-mail address of your site.
 EMAIL_ADDRESS="<user>@${DOMAIN}"
 

.github/workflows/tests.yml

@@ -19,7 +19,7 @@ jobs:
         with:
           python-version: "3.8"
       - name: "Install dependencies"
-        run: pip install wheel && pip install -e .[dev,csp]
+        run: pip install wheel && pip install -e .[dev,csp,redis,memc]
       - name: "Set up .env file"
         run: cp .env.example .env
       - name: "Lint project"
@@ -76,7 +76,7 @@ jobs:
       - name: "Lint project"
         run: flake8 && isort -q -c --df . && mypy .
       - name: "Run tests"
-        run: py.test
+        run: py.test --cov-report=xml
         if: success()
         env:
           DB_TYPE: mysql
@@ -129,7 +129,7 @@ jobs:
       - name: "Lint project"
         run: flake8 && isort -q -c --df . && mypy .
       - name: "Run tests"
-        run: py.test
+        run: py.test --cov-report=xml
         if: success()
         env:
           DB_TYPE: postgresql

.gitignore

@@ -63,6 +63,10 @@ coverage.xml
 status_auth.txt
 !docker/uwsgi.ini
 
+# Redis stuff
+*.rdb
+*.aof
+
 # MangAdventure stuff
 media/
 static/admin/

MangAdventure/cache.py

@@ -0,0 +1,77 @@
+from hashlib import blake2b
+from pickle import UnpicklingError, dumps, loads
+from secrets import compare_digest
+from typing import Any, Tuple
+
+from django.conf import settings
+from django.core.cache.backends.memcached import PyLibMCCache
+from django.core.cache.backends.redis import (
+    RedisCache, RedisCacheClient, RedisSerializer
+)
+
+
+def _sign_data(data: bytes) -> bytes:
+    return blake2b(
+        data, digest_size=16,
+        key=settings.SECRET_KEY.encode()
+    ).digest()
+
+
+class SignedRedisCache(RedisCache):
+    "A cache binding using redis and signed pickles"
+
+    def __init__(self, *args):
+        super().__init__(*args)
+
+        class _SignedRedisSerializer(RedisSerializer):
+            def dumps(self, obj: Any) -> Any:
+                if type(obj) is int:
+                    return obj
+                data = dumps(obj, self.protocol)
+                return _sign_data(data) + data
+
+            def loads(self, data: Any) -> Any:
+                try:
+                    return int(data)
+                except ValueError:
+                    sig, obj = data[:16], data[16:]
+                    if compare_digest(sig, _sign_data(obj)):
+                        return loads(obj)
+                    raise UnpicklingError('Signatures do not match')
+
+        class _SignedRedisCacheClient(RedisCacheClient):
+            def __init__(self, *args, **kwargs):
+                super().__init__(*args, **kwargs)
+                self._serializer = _SignedRedisSerializer()
+
+        self._class = _SignedRedisCacheClient
+
+
+class SignedPyLibMCCache(PyLibMCCache):
+    "A cache binding using pylibmc and signed pickles"
+
+    def __init__(self, *args):
+        super().__init__(*args)
+
+        def _is_pickle(flag: int) -> bool:
+            return flag & 23 == 1
+
+        class _SignedMCClient(self._lib.Client):
+            def serialize(self, value: Any) -> Tuple[bytes, int]:
+                data, flag = super().serialize(value)
+                if _is_pickle(flag):
+                    return _sign_data(data) + data, flag
+                return data, flag
+
+            def deserialize(self, data: bytes, flag: int) -> Any:
+                if _is_pickle(flag):
+                    sig, obj = data[:16], data[16:]
+                    if compare_digest(sig, _sign_data(obj)):
+                        return loads(obj)
+                    raise UnpicklingError('Signatures do not match')
+                return super().deserialize(data, flag)
+
+        self._class = _SignedMCClient
+
+
+__all__ = ['SignedRedisCache', 'SignedPyLibMCCache']

MangAdventure/middleware.py

@@ -19,6 +19,7 @@ class HttpResponseTooEarly(HttpResponse):
 
 class BaseMiddleware(CommonMiddleware):
     """``CommonMiddleware`` with custom patches."""
+
     def __call__(self, request: HttpRequest) -> HttpResponse:
         """
         Patched to allow :const:`blocked user agents
@@ -41,6 +42,7 @@ def __call__(self, request: HttpRequest) -> HttpResponse:
 
 class PreloadMiddleware:
     """Middleware that allows for preloading resources."""
+
     def __init__(self, get_response: Callable[[HttpRequest], HttpResponse]):
         self.get_response = get_response
 

MangAdventure/settings.py

@@ -87,7 +87,6 @@
     'django.contrib.auth.middleware.AuthenticationMiddleware',
     'django.contrib.messages.middleware.MessageMiddleware',
     'django.middleware.cache.FetchFromCacheMiddleware',
-    'django.middleware.clickjacking.XFrameOptionsMiddleware',
     'django.contrib.redirects.middleware.RedirectFallbackMiddleware',
 ]
 
@@ -117,13 +116,37 @@
 #: Default primary key field type to use.
 DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'
 
-##################
-#    Database    #
-##################
+############################
+#    Database & Caching    #
+############################
 
 #: Database settings dictionary. See :setting:`DATABASES`.
 DATABASES = {'default': env.db('DB_URL', 'sqlite:///db.sqlite3')}
 
+#: Cache settings dictionary. See :setting:`CACHES`
+CACHES = {}
+if find_spec('redis'):  # pragma: no cover
+    CACHES['default'] = {
+        'BACKEND': 'MangAdventure.cache.SignedRedisCache',
+        'LOCATION': env['CACHE_URL'],
+        'KEY_PREFIX': env['NAME']
+    }
+elif find_spec('pylibmc'):  # pragma: no cover
+    CACHES['default'] = {
+        'BACKEND': 'MangAdventure.cache.SignedPyLibMCCache',
+        'LOCATION': env['CACHE_URL'],
+        'KEY_PREFIX': env['NAME']
+    }
+else:
+    __import__('warnings').warn_explicit((
+        "Neither redis-py nor pylibmc were installed. "
+        "Falling back to unsigned local memory cache."
+    ), UserWarning, __file__, 146)
+    CACHES['default'] = {
+        'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
+        'KEY_PREFIX': env['NAME']
+    }
+
 ##################################
 #    Logging & Error Handling    #
 ##################################
@@ -382,8 +405,7 @@
 SESSION_COOKIE_SAMESITE = 'Strict'
 
 if env.bool('HTTPS', True):
-    env.ENV['wsgi.url_scheme'] = 'https'
-    MIDDLEWARE.append('MangAdventure.middleware.PreloadMiddleware')
+    MIDDLEWARE.insert(-2, 'MangAdventure.middleware.PreloadMiddleware')
 
     #: HTTP header/value combination that signifies a secure request.
     #: See :setting:`SECURE_PROXY_SSL_HEADER`.
@@ -560,7 +582,7 @@
     ALLOWED_HOSTS += [f'192.168.1.{i}' for i in range(2, 256)]
     if find_spec('debug_toolbar'):
         INSTALLED_APPS.append('debug_toolbar')
-        MIDDLEWARE.append('debug_toolbar.middleware.DebugToolbarMiddleware')
+        MIDDLEWARE.insert(3, 'debug_toolbar.middleware.DebugToolbarMiddleware')
         TEMPLATES[0]['OPTIONS']['context_processors'].append(
             'django.template.context_processors.debug'
         )

MangAdventure/sitemaps.py

@@ -8,6 +8,7 @@
 
 class MiscSitemap(Sitemap):
     """Sitemap for miscellaneous pages."""
+
     def items(self) -> Tuple[str, ...]:
         """
         Get a tuple of the sitemap's items.

MangAdventure/storage.py

@@ -25,6 +25,7 @@ class CDNStorage(FileSystemStorage):
     .. _weserv: https://images.weserv.nl/docs/
     .. _photon: https://developer.wordpress.com/docs/photon/
     """
+
     def __init__(self, fit: Optional[Tuple[int, int]] = None):
         super().__init__()
         self._cdn = cast(str, settings.CONFIG['USE_CDN']).lower()

MangAdventure/templates/manifest.webmanifest

@@ -1,10 +1,10 @@
 {
     "start_url": "/",
-    "lang": "{{lang}}",
-    "name": "{{name}}",
-    "description": "{{description}}",
-    "background_color": "{{background}}",
-    "theme_color": "{{color}}",
+    "lang": "{{ lang }}",
+    "name": "{{ name }}",
+    "description": "{{ description }}",
+    "background_color": "{{ background }}",
+    "theme_color": "{{ color }}",
     "display": "standalone",
     "orientation": "portrait-primary",
     "prefer_related_applications": false,