added flag to disable capturing POST body, and default to false (#151)

closes #151

Author: beniwohli

CHANGELOG.asciidoc

@@ -24,6 +24,7 @@ https://github.com/elastic/apm-agent-python/compare/v1.0.0\...master[Check the H
  * added options for configuring the amount of context lines that are captured with each frame ({pull}136[#136])
  * added support for tracing queries formatted as http://initd.org/psycopg/docs/sql.html[`psycopg2.sql.SQL`] objects ({pull}148[#148])
  * switched to `time.perf_counter` as timing function on Python 3 ({pull}138[#138])
+ * added option to disable capturing of request body ({pull}151[#151])
  * BREAKING: Several settings and APIs have been renamed ({pull}111[#111], {pull}119[#119], {pull}143[#143]):
  ** The decorator for custom instrumentation, `elasticapm.trace`, is now `elasticapm.capture_span`
  ** The setting `traces_send_frequency` has been renamed to `flush_interval`.

docs/configuration.asciidoc

@@ -317,6 +317,29 @@ We differ between errors and spans, as well as library frames and app frames.
 
 WARNING: Especially for spans, collecting source code can have a large impact on storage use in your Elasticsearch cluster.
 
+[float]
+[[config-capture-body]]
+==== `capture_body`
+
+|============
+| Environment                | Django/Flask   | Default
+| `ELASTIC_APM_CAPTURE_BODY` | `CAPTURE_BODY` | `off`
+|============
+
+For transactions that are HTTP requests,
+the Python agent can optionally capture the request body (e.g. `POST` variables).
+
+Possible values: `errors`, `transactions`, `all`, `off`.
+
+If the request has a body and this setting is disabled, the body will be shown as `[REDACTED]`.
+
+For requests with a content type of `multipart/form-data`,
+any uploaded files will be referenced in a special `_files` key.
+It contains the name of the field, and the name of the uploaded file, if provided.
+
+WARNING: request bodies often contain sensitive values like passwords, credit card numbers etc.
+If your service handles data like this, we advise to only enable this feature with care.
+
 [float]
 [[config-flush-interval]]
 ==== `flush_interval`

elasticapm/conf/__init__.py

@@ -166,6 +166,7 @@ class Config(_ConfigBase):
     source_lines_span_library_frames = _ConfigValue('SOURCE_LINES_SPAN_LIBRARY_FRAMES', type=int, default=0)
     local_var_max_length = _ConfigValue('LOCAL_VAR_MAX_LENGTH', type=int, default=200)
     local_var_list_max_length = _ConfigValue('LOCAL_VAR_LIST_MAX_LENGTH', type=int, default=10)
+    capture_body = _ConfigValue('CAPTURE_BODY', default='off')
     async_mode = _BoolConfigValue('ASYNC_MODE', default=True)
     instrument_django_middleware = _BoolConfigValue('INSTRUMENT_DJANGO_MIDDLEWARE', default=True)
     transactions_ignore_patterns = _ListConfigValue('TRANSACTIONS_IGNORE_PATTERNS', default=[])

elasticapm/contrib/django/client.py

@@ -94,27 +94,10 @@ def get_user_info(self, request):
 
         return user_info
 
-    def get_data_from_request(self, request):
-        if request.method != 'GET':
-            try:
-                if hasattr(request, 'body'):
-                    # Django 1.4+
-                    raw_data = request.body
-                else:
-                    raw_data = request.raw_post_data
-                data = raw_data if raw_data else request.POST
-            except Exception:
-                # assume we had a partial read:
-                data = '<unavailable>'
-        else:
-            data = None
-
-        environ = request.META
-
+    def get_data_from_request(self, request, capture_body=False):
         result = {
-            'body': data,
-            'env': dict(get_environ(environ)),
-            'headers': dict(get_headers(environ)),
+            'env': dict(get_environ(request.META)),
+            'headers': dict(get_headers(request.META)),
             'method': request.method,
             'socket': {
                 'remote_address': request.META.get('REMOTE_ADDR'),
@@ -123,6 +106,22 @@ def get_data_from_request(self, request):
             'cookies': dict(request.COOKIES),
         }
 
+        if request.method not in ('GET', 'HEAD'):
+            content_type = request.META.get('CONTENT_TYPE')
+            if content_type == 'application/x-www-form-urlencoded':
+                data = compat.multidict_to_dict(request.POST)
+            elif content_type.startswith('multipart/form-data'):
+                data = compat.multidict_to_dict(request.POST)
+                if request.FILES:
+                    data['_files'] = {field: file.name for field, file in compat.iteritems(request.FILES)}
+            else:
+                try:
+                    data = request.body
+                except Exception:
+                    data = '<unavailable>'
+
+            result['body'] = data if (capture_body or not data) else '[REDACTED]'
+
         if hasattr(request, 'get_raw_uri'):
             # added in Django 1.9
             url = request.get_raw_uri()
@@ -158,7 +157,8 @@ def capture(self, event_type, request=None, **kwargs):
 
         is_http_request = isinstance(request, HttpRequest)
         if is_http_request:
-            context['request'] = self.get_data_from_request(request)
+            context['request'] = self.get_data_from_request(request,
+                                                            capture_body=self.config.capture_body in ('all', 'errors'))
             context['user'] = self.get_user_info(request)
 
         result = super(DjangoClient, self).capture(event_type, **kwargs)

elasticapm/contrib/django/middleware/__init__.py

@@ -181,7 +181,10 @@ def process_response(self, request, response):
                     transaction_name,
                     request
                 )
-                request_data = lambda: self.client.get_data_from_request(request)
+                request_data = lambda: self.client.get_data_from_request(
+                    request,
+                    capture_body=self.client.config.capture_body in ('all', 'transactions')
+                )
                 response_data = lambda: self.client.get_data_from_response(response)
                 elasticapm.set_context(request_data, 'request')
                 elasticapm.set_context(response_data, 'response')

elasticapm/contrib/flask/__init__.py

@@ -89,7 +89,10 @@ def handle_exception(self, *args, **kwargs):
 
         self.client.capture_exception(
             exc_info=kwargs.get('exc_info'),
-            context={'request': get_data_from_request(request)},
+            context={'request': get_data_from_request(
+                request,
+                capture_body=self.client.config.capture_body in ('errors', 'all')
+            )},
             custom={
                 'app': self.app,
             },
@@ -133,8 +136,11 @@ def request_started(self, app):
     def request_finished(self, app, response):
         rule = request.url_rule.rule if request.url_rule is not None else ""
         rule = build_name_with_http_method_prefix(rule, request)
-        request_data = get_data_from_request(request)
-        response_data = get_data_from_response(response)
+        request_data = lambda: get_data_from_request(
+            request,
+            capture_body=self.client.config.capture_body in ('transactions', 'all')
+        )
+        response_data = lambda: get_data_from_response(response)
         elasticapm.set_context(request_data, 'request')
         elasticapm.set_context(response_data, 'response')
         if response.status_code:

elasticapm/contrib/flask/utils.py

@@ -1,22 +1,11 @@
-try:
-    from urllib.parse import urlencode
-except ImportError:
-    # Python 2
-    from urllib import urlencode
+from werkzeug.exceptions import ClientDisconnected
 
 from elasticapm.utils import compat, get_url_dict
 from elasticapm.utils.wsgi import get_environ, get_headers
 
 
-def get_data_from_request(request):
-    body = None
-    if request.data:
-        body = request.data
-    elif request.form:
-        body = urlencode(request.form)
-
+def get_data_from_request(request, capture_body=False):
     result = {
-        'body': body,
         'env': dict(get_environ(request.environ)),
         'headers': dict(
             get_headers(request.environ),
@@ -28,9 +17,27 @@ def get_data_from_request(request):
         },
         'cookies': request.cookies,
     }
+    if request.method not in ('GET', 'HEAD'):
+        body = None
+        if request.content_type == 'application/x-www-form-urlencoded':
+            body = compat.multidict_to_dict(request.form)
+        elif request.content_type.startswith('multipart/form-data'):
+            body = compat.multidict_to_dict(request.form)
+            if request.files:
+                body['_files'] = {
+                    field: val[0].filename if len(val) == 1 else [f.filename for f in val]
+                    for field, val in compat.iterlists(request.files)
+                }
+        else:
+            try:
+                body = request.data
+            except ClientDisconnected:
+                pass
+
+        if body is not None:
+            result['body'] = body if capture_body else '[REDACTED]'
 
     result['url'] = get_url_dict(request.url)
-
     return result
 
 

elasticapm/utils/compat.py

@@ -67,6 +67,11 @@ def iterkeys(d, **kwargs):
 
     def iteritems(d, **kwargs):
         return d.iteritems(**kwargs)
+
+    # for django.utils.datastructures.MultiValueDict
+    def iterlists(d, **kw):
+        return d.iterlists(**kw)
+
 else:
     import io
     import queue  # noqa F401
@@ -93,13 +98,15 @@ def iterkeys(d, **kwargs):
     def iteritems(d, **kwargs):
         return iter(d.items(**kwargs))
 
+    # for django.utils.datastructures.MultiValueDict
+    def iterlists(d, **kw):
+        return iter(d.lists(**kw))
+
 
 def get_default_library_patters():
     """
     Returns library paths depending on the used platform.
 
-    TODO: ensure that this works correctly on Windows
-
     :return: a list of glob paths
     """
     python_version = platform.python_version_tuple()
@@ -114,3 +121,16 @@ def get_default_library_patters():
         if system == 'Windows':
             return [r'*\lib\*']
         return ['*/lib/python%s.%s/*' % python_version[:2], '*/lib64/python%s.%s/*' % python_version[:2]]
+
+
+def multidict_to_dict(d):
+    """
+    Turns a werkzeug.MultiDict or django.MultiValueDict into a dict with
+    list values
+    :param d: a MultiDict or MultiValueDict instance
+    :return: a dict instance
+    """
+    return dict(
+        (k, v[0] if len(v) == 1 else v)
+        for k, v in iterlists(d)
+    )