Add support for truncating dicts in local vars (#638)

* Add support for truncating dicts in local vars

* Update changelog

* Update CONTRIBUTING.md with new changelog name

* Match style in changelog

* Update CHANGELOG.asciidoc

Co-Authored-By: Brandon Morelli <[email protected]>

* Add reviewer suggestions

* Rename metadata key for dicts to be clearer

* Change all refs to "..."

* I blame newborn sleep deprivation.

* Fix one more instance of "..."

Author: basepi

CONTRIBUTING.md

@@ -101,7 +101,7 @@ should "Squash and merge".
 If you have commit access, the process is as follows:
 
 1. Update the version in `elasticapm/version.py` according to the scale of the change. (major, minor or patch)
-1. Update `CHANGELOG.md`
+1. Update `CHANGELOG.asciidoc`
 1. Commit changes with message `update CHANGELOG and bump version to X.Y.Z` where `X.Y.Z` is the version in `elasticapm/version.py`
 1. Tag the commit with `git tag -a vX.Y.Z`, for example `git tag -a v1.2.3`.
    Copy the changelog for the release to the tag message, removing any leading `#`.

docs/configuration.asciidoc

@@ -327,6 +327,18 @@ With this setting, you can limit the length of resulting string.
 With this setting, you can limit the length of lists in local variables.
 
 
+[float]
+[[config-local-dict-var-max-length]]
+==== `local_var_dict_max_length`
+
+|============
+| Environment                             | Django/Flask                | Default
+| `ELASTIC_APM_LOCAL_VAR_DICT_MAX_LENGTH` | `LOCAL_VAR_DICT_MAX_LENGTH` | `10`
+|============
+
+With this setting, you can limit the length of dicts in local variables.
+
+
 [float]
 [[config-source-lines-error-app-frames]]
 ==== `source_lines_error_app_frames`

elasticapm/base.py

@@ -172,6 +172,7 @@ def __init__(self, config=None, **inline):
                         v,
                         list_length=self.config.local_var_list_max_length,
                         string_length=self.config.local_var_max_length,
+                        dict_length=self.config.local_var_dict_max_length,
                     ),
                     local_var,
                 ),
@@ -431,6 +432,7 @@ def _build_msg_for_logging(
                         v,
                         list_length=self.config.local_var_list_max_length,
                         string_length=self.config.local_var_max_length,
+                        dict_length=self.config.local_var_dict_max_length,
                     ),
                     local_var,
                 ),

elasticapm/conf/__init__.py

@@ -318,6 +318,7 @@ class Config(_ConfigBase):
     source_lines_span_library_frames = _ConfigValue("SOURCE_LINES_SPAN_LIBRARY_FRAMES", type=int, default=0)
     local_var_max_length = _ConfigValue("LOCAL_VAR_MAX_LENGTH", type=int, default=200)
     local_var_list_max_length = _ConfigValue("LOCAL_VAR_LIST_MAX_LENGTH", type=int, default=10)
+    local_var_dict_max_length = _ConfigValue("LOCAL_VAR_DICT_MAX_LENGTH", type=int, default=10)
     capture_body = _ConfigValue("CAPTURE_BODY", default="off")
     async_mode = _BoolConfigValue("ASYNC_MODE", default=True)
     instrument_django_middleware = _BoolConfigValue("INSTRUMENT_DJANGO_MIDDLEWARE", default=True)

elasticapm/events.py

@@ -105,6 +105,7 @@ def capture(client, exc_info=None, **kwargs):
                         val,
                         list_length=client.config.local_var_list_max_length,
                         string_length=client.config.local_var_max_length,
+                        dict_length=client.config.local_var_dict_max_length,
                     ),
                     local_var,
                 ),

elasticapm/processors.py

@@ -245,6 +245,11 @@ def _sanitize(key, value):
     if isinstance(value, compat.string_types) and any(pattern.match(value) for pattern in SANITIZE_VALUE_PATTERNS):
         return MASK
 
+    if isinstance(value, dict):
+        # varmap will call _sanitize on each k:v pair of the dict, so we don't
+        # have to do anything with dicts here
+        return value
+
     if not key:  # key can be a NoneType
         return value
 

elasticapm/utils/__init__.py

@@ -61,7 +61,7 @@ def varmap(func, var, context=None, name=None):
         return func(name, "<...>")
     context.add(objid)
     if isinstance(var, dict):
-        ret = dict((k, varmap(func, v, context, k)) for k, v in compat.iteritems(var))
+        ret = func(name, dict((k, varmap(func, v, context, k)) for k, v in compat.iteritems(var)))
     elif isinstance(var, (list, tuple)):
         ret = func(name, [varmap(func, f, context, name) for f in var])
     else:

elasticapm/utils/encoding.py

@@ -32,6 +32,7 @@
 
 
 import datetime
+import itertools
 import uuid
 from decimal import Decimal
 
@@ -178,15 +179,41 @@ def to_string(value):
         return to_unicode(value).encode("utf-8")
 
 
-def shorten(var, list_length=50, string_length=200):
+def shorten(var, list_length=50, string_length=200, dict_length=50):
+    """
+    Shorten a given variable based on configurable maximum lengths, leaving
+    breadcrumbs in the object to show that it was shortened.
+
+    For strings, truncate the string to the max length, and append "..." so
+    the user knows data was lost.
+
+    For lists, truncate the list to the max length, and append two new strings
+    to the list: "..." and "(<x> more elements)" where <x> is the number of
+    elements removed.
+
+    For dicts, truncate the dict to the max length (based on number of key/value
+    pairs) and add a new (key, value) pair to the dict:
+    ("...", "(<x> more elements)") where <x> is the number of key/value pairs
+    removed.
+
+    :param var: Variable to be shortened
+    :param list_length: Max length (in items) of lists
+    :param string_length: Max length (in characters) of strings
+    :param dict_length: Max length (in key/value pairs) of dicts
+    :return: Shortened variable
+    """
     var = transform(var)
     if isinstance(var, compat.string_types) and len(var) > string_length:
         var = var[: string_length - 3] + "..."
     elif isinstance(var, (list, tuple, set, frozenset)) and len(var) > list_length:
         # TODO: we should write a real API for storing some metadata with vars when
         # we get around to doing ref storage
-        # TODO: when we finish the above, we should also implement this for dicts
         var = list(var)[:list_length] + ["...", "(%d more elements)" % (len(var) - list_length,)]
+    elif isinstance(var, dict) and len(var) > dict_length:
+        trimmed_tuples = [(k, v) for (k, v) in itertools.islice(compat.iteritems(var), dict_length)]
+        if "<truncated>" not in var:
+            trimmed_tuples += [("<truncated>", "(%d more elements)" % (len(var) - dict_length))]
+        var = dict(trimmed_tuples)
     return var
 
 

tests/processors/tests.py

@@ -231,6 +231,11 @@ def test_sanitize_credit_card_with_spaces():
     assert result == processors.MASK
 
 
+def test_sanitize_dict():
+    result = processors._sanitize("foo", {1: 2})
+    assert result == {1: 2}
+
+
 def test_non_utf8_encoding(http_test_data):
     broken = compat.b("broken=") + u"aéöüa".encode("latin-1")
     http_test_data["context"]["request"]["url"]["search"] = broken

tests/utils/encoding/tests.py

@@ -227,6 +227,12 @@ def test_shorten_tuple():
     #     assert result[-1] == '(450 more elements)'
 
 
+def test_shorten_dict():
+    result = shorten({k: k for k in range(500)}, dict_length=50)
+    assert len(result) == 51
+    assert result["<truncated>"] == "(450 more elements)"
+
+
 def test_enforce_label_format():
     class MyObj(object):
         def __str__(self):