Merge pull request #70 from palazzem/django-cache

[django] instrument the Django cache framework

Author: Emanuele Palazzetti

ddtrace/contrib/django/apps.py

@@ -6,6 +6,7 @@
 # project
 from .db import patch_db
 from .conf import settings
+from .cache import patch_cache
 from .templates import patch_template
 
 from ...ext import AppTypes
@@ -28,9 +29,9 @@ def ready(self):
 
             # define the service details
             tracer.set_service_info(
-                service=settings.DEFAULT_SERVICE,
                 app='django',
                 app_type=AppTypes.web,
+                service=settings.DEFAULT_SERVICE,
             )
 
             # trace Django internals
@@ -43,3 +44,8 @@ def ready(self):
                 patch_template(tracer)
             except Exception:
                 log.exception('error patching Django template rendering')
+
+            try:
+                patch_cache(tracer)
+            except Exception:
+                log.exception('error patching Django cache')

ddtrace/contrib/django/cache.py

@@ -0,0 +1,91 @@
+import logging
+
+from functools import wraps
+
+from django.conf import settings as django_settings
+
+from .conf import settings, import_from_string
+from .utils import quantize_key_values, _resource_from_cache_prefix
+
+
+log = logging.getLogger(__name__)
+
+# code instrumentation
+DATADOG_NAMESPACE = '__datadog_original_{method}'
+TRACED_METHODS = [
+    'get',
+    'set',
+    'add',
+    'delete',
+    'incr',
+    'decr',
+    'get_many',
+    'set_many',
+    'delete_many',
+]
+
+# standard tags
+TYPE = 'cache'
+CACHE_BACKEND = 'django.cache.backend'
+CACHE_COMMAND_KEY = 'django.cache.key'
+
+
+def patch_cache(tracer):
+    """
+    Function that patches the inner cache system. Because the cache backend
+    can have different implementations and connectors, this function must
+    handle all possible interactions with the Django cache. What follows
+    is currently traced:
+        * in-memory cache
+        * the cache client wrapper that could use any of the common
+          Django supported cache servers (Redis, Memcached, Database, Custom)
+    """
+    # discover used cache backends
+    cache_backends = [cache['BACKEND'] for cache in django_settings.CACHES.values()]
+
+    def _trace_operation(fn, method_name):
+        """
+        Return a wrapped function that traces a cache operation
+        """
+        @wraps(fn)
+        def wrapped(self, *args, **kwargs):
+            # get the original function method
+            method = getattr(self, DATADOG_NAMESPACE.format(method=method_name))
+            with tracer.trace('django.cache',
+                    span_type=TYPE, service=settings.DEFAULT_SERVICE) as span:
+                # update the resource name and tag the cache backend
+                span.resource = _resource_from_cache_prefix(method_name, self)
+                cache_backend = '{}.{}'.format(self.__module__, self.__class__.__name__)
+                span.set_tag(CACHE_BACKEND, cache_backend)
+
+                if args:
+                    keys = quantize_key_values(args[0])
+                    span.set_tag(CACHE_COMMAND_KEY, keys)
+
+                return method(*args, **kwargs)
+        return wrapped
+
+    def _wrap_method(cls, method_name):
+        """
+        For the given class, wraps the method name with a traced operation
+        so that the original method is executed, while the span is properly
+        created
+        """
+        # check if the backend owns the given bounded method
+        if not hasattr(cls, method_name):
+            return
+
+        # prevent patching each backend's method more than once
+        if hasattr(cls, DATADOG_NAMESPACE.format(method=method_name)):
+            log.debug('{} already traced'.format(method_name))
+        else:
+            method = getattr(cls, method_name)
+            setattr(cls, DATADOG_NAMESPACE.format(method=method_name), method)
+            setattr(cls, method_name, _trace_operation(method, method_name))
+
+    # trace all backends
+    for cache_module in cache_backends:
+        cache = import_from_string(cache_module, cache_module)
+
+        for method in TRACED_METHODS:
+            _wrap_method(cache, method)

ddtrace/contrib/django/conf.py

@@ -102,7 +102,7 @@ def __getattr__(self, attr):
         return val
 
     def __check_user_settings(self, user_settings):
-        SETTINGS_DOC = 'http://pypi.datadoghq.com/trace-dev/docs/#module-ddtrace.contrib.django'
+        SETTINGS_DOC = 'http://pypi.datadoghq.com/trace/docs/#module-ddtrace.contrib.django'
         for setting in REMOVED_SETTINGS:
             if setting in user_settings:
                 raise RuntimeError(

ddtrace/contrib/django/utils.py

@@ -0,0 +1,26 @@
+def _resource_from_cache_prefix(resource, cache):
+    """
+    Combine the resource name with the cache prefix (if any)
+    """
+    if getattr(cache, "key_prefix", None):
+        name = "{} {}".format(resource, cache.key_prefix)
+    else:
+        name = resource
+
+    # enforce lowercase to make the output nicer to read
+    return name.lower()
+
+
+def quantize_key_values(key):
+    """
+    Used in the Django trace operation method, it ensures that if a dict
+    with values is used, we removes the values from the span meta
+    attributes. For example::
+
+        >>> quantize_key_values({'key', 'value'})
+        # returns ['key']
+    """
+    if isinstance(key, dict):
+        return key.keys()
+
+    return key

tests/contrib/django/app/settings.py

@@ -15,6 +15,34 @@
     }
 }
 
+CACHES = {
+    'default': {
+        'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
+        'LOCATION': 'unique-snowflake',
+    },
+    'redis': {
+        'BACKEND': 'django_redis.cache.RedisCache',
+        'LOCATION': 'redis://127.0.0.1:56379/1',
+    },
+    'pylibmc': {
+        'BACKEND': 'django.core.cache.backends.memcached.PyLibMCCache',
+        'LOCATION': '127.0.0.1:51211',
+    },
+    'python_memcached': {
+        'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
+        'LOCATION': '127.0.0.1:51211',
+    },
+    'django_pylibmc': {
+        'BACKEND': 'django_pylibmc.memcached.PyLibMCCache',
+        'LOCATION': '127.0.0.1:51211',
+        'BINARY': True,
+        'OPTIONS': {
+            'tcp_nodelay': True,
+            'ketama': True
+        }
+    },
+}
+
 SITE_ID = 1
 SECRET_KEY = 'not_very_secret_in_tests'
 USE_I18N = True

tests/contrib/django/app/templates/cached_list.html

@@ -0,0 +1,7 @@
+{% load cache %}
+
+{% cache 60 users_list %}
+    {% for user in object_list %}
+        {{ user }}
+    {% endfor %}
+{% endcache %}

tests/contrib/django/app/views.py

@@ -3,15 +3,21 @@
 """
 from django.http import HttpResponse
 from django.conf.urls import url
-from django.views.generic import ListView, TemplateView
 from django.contrib.auth.models import User
+from django.views.generic import ListView, TemplateView
+from django.views.decorators.cache import cache_page
 
 
 class UserList(ListView):
     model = User
     template_name = 'users_list.html'
 
 
+class TemplateCachedUserList(ListView):
+    model = User
+    template_name = 'cached_list.html'
+
+
 class ForbiddenView(TemplateView):
     def get(self, request, *args, **kwargs):
         return HttpResponse(status=403)
@@ -20,5 +26,7 @@ def get(self, request, *args, **kwargs):
 # use this url patterns for tests
 urlpatterns = [
     url(r'^users/$', UserList.as_view(), name='users-list'),
+    url(r'^cached-template/$', TemplateCachedUserList.as_view(), name='cached-template-list'),
+    url(r'^cached-users/$', cache_page(60)(UserList.as_view()), name='cached-users-list'),
     url(r'^fail-view/$', ForbiddenView.as_view(), name='forbidden-view'),
 ]