Use django cache with large-image (#32)

* Use django cache with large-image

* Linting

* Remove print statements

* Remove rest caching

* Add test to validate large-image uses cache

* Handle improper django configuration

* Update README

* Bump large image

Author: banesullivan

README.md

@@ -85,8 +85,9 @@ Support for any storage backend:
 
 Miscellaneous:
 - Admin interface widget for viewing image tiles.
-- Caching - tile sources are cached for rapid file re-opening
-  - tiles and thumbnails are cached to prevent recreating these data on multiple requests
+- Caching
+  - image tiles and thumbnails are cached to prevent recreating these data on multiple requests
+  - utilizes the [Django cache framework](https://docs.djangoproject.com/en/4.0/topics/cache/). Specify a named cache to use with the `LARGE_IMAGE_CACHE_NAME` setting.
 - Easily extensible SSR templates for tile viewing with CesiumJS and GeoJS
 - OpenAPI specification
 
@@ -116,7 +117,7 @@ production environments. To install our GDAL wheel, use:
 pip install \
   --find-links https://girder.github.io/large_image_wheels \
   django-large-image \
-  'large-image[gdal,pil]>=1.14'
+  'large-image[gdal,pil]>=1.15'
 ```
 
 ### 🐍 Conda

django_large_image/apps.py

@@ -1,7 +1,6 @@
 import logging
 
 from django.apps import AppConfig
-from django.conf import settings
 import large_image
 
 logger = logging.getLogger(__name__)
@@ -13,20 +12,7 @@ class DjangoLargeImageConfig(AppConfig):
     default_auto_field = 'django.db.models.BigAutoField'
 
     def ready(self):
-        # Set up memcached with large_image
-        if hasattr(settings, 'MEMCACHED_URL') and settings.MEMCACHED_URL:
-            large_image.config.setConfig('cache_memcached_url', settings.MEMCACHED_URL)
-            if (
-                hasattr(settings, 'MEMCACHED_USERNAME')
-                and settings.MEMCACHED_USERNAME
-                and hasattr(settings, 'MEMCACHED_PASSWORD')
-                and settings.MEMCACHED_PASSWORD
-            ):
-                large_image.config.setConfig(
-                    'cache_memcached_username', settings.MEMCACHED_USERNAME
-                )
-                large_image.config.setConfig(
-                    'cache_memcached_password', settings.MEMCACHED_PASSWORD
-                )
-            large_image.config.setConfig('cache_backend', 'memcached')
-            logger.info('large_image is configured for memcached.')
+        # Set up cache with large_image
+        # This isn't necessary but it makes sure we always default
+        #   to the django cache if others are available
+        large_image.config.setConfig('cache_backend', 'django')

django_large_image/cache.py

@@ -0,0 +1,72 @@
+import threading
+
+from django.conf import settings
+from django.core.cache import caches
+from django.core.exceptions import ImproperlyConfigured
+from large_image.cache_util.base import BaseCache
+from large_image.exceptions import TileCacheConfigurationError
+
+
+class DjangoCache(BaseCache):
+    """Use Django cache as the backing cache for large-image."""
+
+    def __init__(self, cache, getsizeof=None):
+        super().__init__(0, getsizeof=getsizeof)
+        self._django_cache = cache
+
+    def __repr__(self):  # pragma: no cover
+        return f'DjangoCache<{repr(self._django_cache._alias)}>'
+
+    def __iter__(self):  # pragma: no cover
+        # return invalid iter
+        return None
+
+    def __len__(self):  # pragma: no cover
+        # return invalid length
+        return -1
+
+    def __contains__(self, key):
+        hashed_key = self._hashKey(key)
+        return self._django_cache.__contains__(hashed_key)
+
+    def __delitem__(self, key):
+        hashed_key = self._hashKey(key)
+        return self._django_cache.delete(hashed_key)
+
+    def __getitem__(self, key):
+        hashed_key = self._hashKey(key)
+        value = self._django_cache.get(hashed_key)
+        if value is None:
+            return self.__missing__(key)
+        return value
+
+    def __setitem__(self, key, value):
+        hashed_key = self._hashKey(key)
+        # TODO: do we want to use `add` instead to add a key only if it doesn’t already exist
+        return self._django_cache.set(hashed_key, value)
+
+    @property
+    def curritems(self):  # pragma: no cover
+        raise NotImplementedError
+
+    @property
+    def currsize(self):  # pragma: no cover
+        raise NotImplementedError
+
+    @property
+    def maxsize(self):  # pragma: no cover
+        raise NotImplementedError
+
+    def clear(self):
+        self._django_cache.clear()
+
+    @staticmethod
+    def getCache():  # noqa: N802
+        try:
+            name = getattr(settings, 'LARGE_IMAGE_CACHE_NAME', 'default')
+            dajngo_cache = caches[name]
+        except ImproperlyConfigured:
+            raise TileCacheConfigurationError
+        cache_lock = threading.Lock()
+        cache = DjangoCache(dajngo_cache)
+        return cache, cache_lock

django_large_image/rest/base.py

@@ -9,8 +9,6 @@
 
 from django_large_image import tilesource, utilities
 
-CACHE_TIMEOUT = 60 * 60 * 2
-
 
 class LargeImageMixinBase:
     def get_path(self, request: Request, pk: int = None) -> Union[str, pathlib.Path]:

django_large_image/rest/data.py

@@ -1,6 +1,4 @@
 from django.http import HttpResponse
-from django.utils.decorators import method_decorator
-from django.views.decorators.cache import cache_page
 from drf_yasg.utils import swagger_auto_schema
 from rest_framework.decorators import action
 from rest_framework.exceptions import ValidationError
@@ -9,7 +7,7 @@
 
 from django_large_image import tilesource, utilities
 from django_large_image.rest import params
-from django_large_image.rest.base import CACHE_TIMEOUT, LargeImageMixinBase
+from django_large_image.rest.base import LargeImageMixinBase
 from django_large_image.rest.renderers import image_data_renderers, image_renderers
 
 thumbnail_summary = 'Returns thumbnail of full image.'
@@ -23,7 +21,6 @@
 
 
 class DataMixin(LargeImageMixinBase):
-    @method_decorator(cache_page(CACHE_TIMEOUT))
     @swagger_auto_schema(
         method='GET',
         operation_summary=thumbnail_summary,

django_large_image/rest/tiles.py

@@ -1,6 +1,4 @@
 from django.http import HttpResponse
-from django.utils.decorators import method_decorator
-from django.views.decorators.cache import cache_page
 from drf_yasg.utils import swagger_auto_schema
 from large_image.exceptions import TileSourceXYZRangeError
 from rest_framework.decorators import action
@@ -10,7 +8,7 @@
 
 from django_large_image import tilesource
 from django_large_image.rest import params
-from django_large_image.rest.base import CACHE_TIMEOUT, LargeImageMixinBase
+from django_large_image.rest.base import LargeImageMixinBase
 from django_large_image.rest.renderers import image_renderers
 from django_large_image.rest.serializers import TileMetadataSerializer
 
@@ -35,7 +33,6 @@ def tiles_metadata(self, request: Request, pk: int = None) -> Response:
         serializer = TileMetadataSerializer(source)
         return Response(serializer.data)
 
-    @method_decorator(cache_page(CACHE_TIMEOUT))
     @swagger_auto_schema(
         method='GET',
         operation_summary=tile_summary,

project/example/core/tests/conftest.py

@@ -62,3 +62,8 @@ def lonely_header_file() -> models.ImageFile:
         file__filename='envi_rgbsmall_bip.hdr',
         file__from_path=datastore.fetch('envi_rgbsmall_bip.hdr'),
     )
+
+
+@pytest.fixture
+def geotiff_path():
+    return datastore.fetch('rgb_geotiff.tiff')

project/example/core/tests/test_cache.py

@@ -0,0 +1,36 @@
+from large_image.cache_util.base import BaseCache
+import pytest
+
+from django_large_image import tilesource
+from django_large_image.cache import DjangoCache
+
+
+@pytest.fixture
+def cache_miss_counter():
+    class Counter:
+        def __init__(self):
+            self.count = 0
+
+        def reset(self):
+            self.count = 0
+
+    counter = Counter()
+
+    def missing(*args, **kwargs):
+        counter.count += 1
+        BaseCache.__missing__(*args, **kwargs)
+
+    original = DjangoCache.__missing__
+    DjangoCache.__missing__ = missing
+    yield counter
+    DjangoCache.__missing__ = original
+
+
+def test_tile(geotiff_path, cache_miss_counter):
+    source = tilesource.get_tilesource_from_path(geotiff_path)
+    cache_miss_counter.reset()
+    # Check size of cache
+    _ = source.getTile(0, 0, 0, encoding='PNG')
+    assert cache_miss_counter.count == 1
+    _ = source.getTile(0, 0, 0, encoding='PNG')
+    assert cache_miss_counter.count == 1

project/setup.py

@@ -50,7 +50,7 @@
         'django-s3-file-field[boto3]',
         'gunicorn',
         'django-large-image',
-        'large-image[gdal,pil,ometiff,converter,vips,openslide,openjpeg]>=1.14',
+        'large-image[gdal,pil,ometiff,converter,vips,openslide,openjpeg]>=1.15',
         'pooch',
     ],
     extras_require={

setup.py

@@ -42,12 +42,15 @@
         'djangorestframework',
         'drf-yasg',
         'filelock',
-        'large-image>=1.14',
+        'large-image>=1.15',
     ],
     extras_require={
         'colormaps': [
             'matplotlib',
             'cmocean',
         ],
     },
+    entry_points={
+        'large_image.cache': ['django = django_large_image.cache:DjangoCache'],
+    },
 )