Config sanitize field names (#860)

* feat: configurable SANITIZE_FIELD_NAMES

* tests: tests for configurable SANITIZE_FIELD_NAMES

* refactor: wrap a class around field name operations

* chore: fixes

* Send client, not transport, to processors

* fix: update sanitize_stacktrace_locals to use config fields

* Cleanup processors

Co-authored-by: Colton Myers <[email protected]>

* fix: pass sanitize_field_names to _sanitize_string

* chore: cleanup, tests, and doc updates

* chore: reorder docs statements

Co-authored-by: Colton Myers <[email protected]>

Author: AumitLeon

docs/configuration.asciidoc

@@ -636,6 +636,28 @@ For more information, see <<sanitizing-data, Sanitizing Data>>.
 
 WARNING: We recommend to always include the default set of validators if you customize this setting.
 
+[float]
+[[config-sanitize-field-names]]
+==== `sanitize_field_names`
+
+[options="header"]
+|============
+| Environment              | Django/Flask | Default
+| `ELASTIC_APM_SANITIZE_FIELD_NAMES` | `SANITIZE_FIELD_NAMES` | `['authorization',
+                                                                  'password',
+                                                                  'secret',
+                                                                  'passwd',
+                                                                  'token',
+                                                                  'api_key',
+                                                                  'access_token',
+                                                                  'sessionid']`
+|============
+
+A list of field names to mask when using processors.
+For more information, see <<sanitizing-data, Sanitizing Data>>.
+
+WARNING: We recommend to always include the default set of field names if you customize this setting.
+
 
 [float]
 [[config-transaction-sample-rate]]

docs/sanitizing-data.asciidoc

@@ -48,3 +48,28 @@ ELASTIC_APM = {
 ----
 
 NOTE: We recommend to use the above list of processors that sanitize passwords and secrets in different places of the event object.
+
+The default set of processors sanitize fields based on a set of defaults defined in `elasticapm.conf.constants`. This set can be configured with the `SANATIZE_FIELD_NAMES` configuration option. For example, if your application produces a sensitive field called `My-Sensitive-Field`, the default processors can be used to automatically sanitize this field. You can specify what fields to santize within default processors like this:
+
+[source,python]
+----
+ELASTIC_APM = {
+    'APP_NAME': '<APP-NAME>',
+    'SECRET_TOKEN': '<SECRET-TOKEN>',
+    'SANITIZE_FIELD_NAMES': (
+        'My-Sensitive-Field',
+        'authorization',
+        'password',
+        'secret',
+        'passwd',
+        'token',
+        'api_key',
+        'access_token',
+        'sessionid',
+    ),
+}
+----
+
+NOTE: We recommend to use the above list of fields to sanitize various parts of the event object in addition to your specified fields.
+
+When choosing fields names to sanitize, you can specify values that will match certain wildcards. For example, passing `base` as a field name to be sanitized will also sanitize all fields whose names match the regex pattern `\*base*`.

elasticapm/conf/__init__.py

@@ -35,6 +35,7 @@
 import socket
 import threading
 
+from elasticapm.conf.constants import BASE_SANITIZE_FIELD_NAMES
 from elasticapm.utils import compat, starmatch_to_regex
 from elasticapm.utils.logging import get_logger
 from elasticapm.utils.threading import IntervalTimer, ThreadManager
@@ -286,6 +287,7 @@ class Config(_ConfigBase):
             "elasticapm.processors.sanitize_http_request_body",
         ],
     )
+    sanitize_field_names = _ListConfigValue("SANITIZE_FIELD_NAMES", default=BASE_SANITIZE_FIELD_NAMES)
     metrics_sets = _ListConfigValue(
         "METRICS_SETS",
         default=[

elasticapm/conf/constants.py

@@ -58,6 +58,17 @@
 
 HARDCODED_PROCESSORS = ["elasticapm.processors.add_context_lines_to_frames"]
 
+BASE_SANITIZE_FIELD_NAMES = [
+    "authorization",
+    "password",
+    "secret",
+    "passwd",
+    "token",
+    "api_key",
+    "access_token",
+    "sessionid",
+]
+
 try:
     # Python 2
     LABEL_TYPES = (bool, int, long, float, decimal.Decimal)

elasticapm/processors.py

@@ -33,22 +33,17 @@
 import warnings
 from collections import defaultdict
 
-from elasticapm.conf.constants import ERROR, MASK, SPAN, TRANSACTION
+from elasticapm.conf.constants import BASE_SANITIZE_FIELD_NAMES, ERROR, MASK, SPAN, TRANSACTION
 from elasticapm.utils import compat, varmap
 from elasticapm.utils.encoding import force_text
 from elasticapm.utils.stacks import get_lines_from_file
 
-SANITIZE_FIELD_NAMES = frozenset(
-    ["authorization", "password", "secret", "passwd", "token", "api_key", "access_token", "sessionid"]
-)
-
 SANITIZE_VALUE_PATTERNS = [re.compile(r"^[- \d]{16,19}$")]  # credit card numbers, with or without spacers
 
 
 def for_events(*events):
     """
     :param events: list of event types
-
     Only calls wrapped function if given event_type is in list of events
     """
     events = set(events)
@@ -64,7 +59,6 @@ def wrap(func):
 def remove_http_request_body(client, event):
     """
     Removes request.body from context
-
     :param client: an ElasticAPM client
     :param event: a transaction or error event
     :return: The modified event
@@ -78,7 +72,6 @@ def remove_http_request_body(client, event):
 def remove_stacktrace_locals(client, event):
     """
     Removes local variables from any frames.
-
     :param client: an ElasticAPM client
     :param event: a transaction or error event
     :return: The modified event
@@ -91,15 +84,14 @@ def remove_stacktrace_locals(client, event):
 def sanitize_stacktrace_locals(client, event):
     """
     Sanitizes local variables in all frames
-
     :param client: an ElasticAPM client
     :param event: a transaction or error event
     :return: The modified event
     """
 
     def func(frame):
         if "vars" in frame:
-            frame["vars"] = varmap(_sanitize, frame["vars"])
+            frame["vars"] = varmap(_sanitize, frame["vars"], sanitize_field_names=client.config.sanitize_field_names)
 
     return _process_stack_frames(event, func)
 
@@ -108,7 +100,6 @@ def func(frame):
 def sanitize_http_request_cookies(client, event):
     """
     Sanitizes http request cookies
-
     :param client: an ElasticAPM client
     :param event: a transaction or error event
     :return: The modified event
@@ -117,14 +108,18 @@ def sanitize_http_request_cookies(client, event):
     # sanitize request.cookies dict
     try:
         cookies = event["context"]["request"]["cookies"]
-        event["context"]["request"]["cookies"] = varmap(_sanitize, cookies)
+        event["context"]["request"]["cookies"] = varmap(
+            _sanitize, cookies, sanitize_field_names=client.config.sanitize_field_names
+        )
     except (KeyError, TypeError):
         pass
 
     # sanitize request.header.cookie string
     try:
         cookie_string = event["context"]["request"]["headers"]["cookie"]
-        event["context"]["request"]["headers"]["cookie"] = _sanitize_string(cookie_string, "; ", "=")
+        event["context"]["request"]["headers"]["cookie"] = _sanitize_string(
+            cookie_string, "; ", "=", sanitize_field_names=client.config.sanitize_field_names
+        )
     except (KeyError, TypeError):
         pass
     return event
@@ -140,7 +135,9 @@ def sanitize_http_response_cookies(client, event):
     """
     try:
         cookie_string = event["context"]["response"]["headers"]["set-cookie"]
-        event["context"]["response"]["headers"]["set-cookie"] = _sanitize_string(cookie_string, ";", "=")
+        event["context"]["response"]["headers"]["set-cookie"] = _sanitize_string(
+            cookie_string, ";", "=", sanitize_field_names=client.config.sanitize_field_names
+        )
     except (KeyError, TypeError):
         pass
     return event
@@ -150,22 +147,25 @@ def sanitize_http_response_cookies(client, event):
 def sanitize_http_headers(client, event):
     """
     Sanitizes http request/response headers
-
     :param client: an ElasticAPM client
     :param event: a transaction or error event
     :return: The modified event
     """
     # request headers
     try:
         headers = event["context"]["request"]["headers"]
-        event["context"]["request"]["headers"] = varmap(_sanitize, headers)
+        event["context"]["request"]["headers"] = varmap(
+            _sanitize, headers, sanitize_field_names=client.config.sanitize_field_names
+        )
     except (KeyError, TypeError):
         pass
 
     # response headers
     try:
         headers = event["context"]["response"]["headers"]
-        event["context"]["response"]["headers"] = varmap(_sanitize, headers)
+        event["context"]["response"]["headers"] = varmap(
+            _sanitize, headers, sanitize_field_names=client.config.sanitize_field_names
+        )
     except (KeyError, TypeError):
         pass
 
@@ -176,14 +176,15 @@ def sanitize_http_headers(client, event):
 def sanitize_http_wsgi_env(client, event):
     """
     Sanitizes WSGI environment variables
-
     :param client: an ElasticAPM client
     :param event: a transaction or error event
     :return: The modified event
     """
     try:
         env = event["context"]["request"]["env"]
-        event["context"]["request"]["env"] = varmap(_sanitize, env)
+        event["context"]["request"]["env"] = varmap(
+            _sanitize, env, sanitize_field_names=client.config.sanitize_field_names
+        )
     except (KeyError, TypeError):
         pass
     return event
@@ -193,7 +194,6 @@ def sanitize_http_wsgi_env(client, event):
 def sanitize_http_request_querystring(client, event):
     """
     Sanitizes http request query string
-
     :param client: an ElasticAPM client
     :param event: a transaction or error event
     :return: The modified event
@@ -203,7 +203,9 @@ def sanitize_http_request_querystring(client, event):
     except (KeyError, TypeError):
         return event
     if "=" in query_string:
-        sanitized_query_string = _sanitize_string(query_string, "&", "=")
+        sanitized_query_string = _sanitize_string(
+            query_string, "&", "=", sanitize_field_names=client.config.sanitize_field_names
+        )
         full_url = event["context"]["request"]["url"]["full"]
         event["context"]["request"]["url"]["search"] = sanitized_query_string
         event["context"]["request"]["url"]["full"] = full_url.replace(query_string, sanitized_query_string)
@@ -216,7 +218,6 @@ def sanitize_http_request_body(client, event):
     Sanitizes http request body. This only works if the request body
     is a query-encoded string. Other types (e.g. JSON) are not handled by
     this sanitizer.
-
     :param client: an ElasticAPM client
     :param event: a transaction or error event
     :return: The modified event
@@ -226,7 +227,9 @@ def sanitize_http_request_body(client, event):
     except (KeyError, TypeError):
         return event
     if "=" in body:
-        sanitized_query_string = _sanitize_string(body, "&", "=")
+        sanitized_query_string = _sanitize_string(
+            body, "&", "=", sanitize_field_names=client.config.sanitize_field_names
+        )
         event["context"]["request"]["body"] = sanitized_query_string
     return event
 
@@ -267,7 +270,12 @@ def mark_in_app_frames(client, event):
     return event
 
 
-def _sanitize(key, value):
+def _sanitize(key, value, **kwargs):
+    if "sanitize_field_names" in kwargs:
+        sanitize_field_names = frozenset(kwargs["sanitize_field_names"])
+    else:
+        sanitize_field_names = frozenset(BASE_SANITIZE_FIELD_NAMES)
+
     if value is None:
         return
 
@@ -283,27 +291,28 @@ def _sanitize(key, value):
         return value
 
     key = key.lower()
-    for field in SANITIZE_FIELD_NAMES:
+    for field in sanitize_field_names:
         if field in key:
             # store mask as a fixed length for security
             return MASK
     return value
 
 
-def _sanitize_string(unsanitized, itemsep, kvsep):
+def _sanitize_string(unsanitized, itemsep, kvsep, sanitize_field_names=BASE_SANITIZE_FIELD_NAMES):
     """
     sanitizes a string that contains multiple key/value items
     :param unsanitized: the unsanitized string
     :param itemsep: string that separates items
     :param kvsep: string that separates key from value
+    :param sanitize_field_names: field names to pass to _sanitize
     :return: a sanitized string
     """
     sanitized = []
     kvs = unsanitized.split(itemsep)
     for kv in kvs:
         kv = kv.split(kvsep)
         if len(kv) == 2:
-            sanitized.append((kv[0], _sanitize(kv[0], kv[1])))
+            sanitized.append((kv[0], _sanitize(kv[0], kv[1], sanitize_field_names=sanitize_field_names)))
         else:
             sanitized.append(kv)
     return itemsep.join(kvsep.join(kv) for kv in sanitized)

elasticapm/transport/base.py

@@ -176,7 +176,7 @@ def _process_event(self, event_type, data):
         # Run the data through processors
         for processor in self._processors:
             if not hasattr(processor, "event_types") or event_type in processor.event_types:
-                data = processor(self, data)
+                data = processor(self.client, data)
                 if not data:
                     logger.debug(
                         "Dropped event of type %s due to processor %s.%s",

elasticapm/utils/__init__.py

@@ -48,7 +48,7 @@
 default_ports = {"https": 443, "http": 80, "postgresql": 5432, "mysql": 3306, "mssql": 1433}
 
 
-def varmap(func, var, context=None, name=None):
+def varmap(func, var, context=None, name=None, **kwargs):
     """
     Executes ``func(key_name, value)`` on all values,
     recursively discovering dict and list scoped
@@ -58,14 +58,14 @@ def varmap(func, var, context=None, name=None):
         context = set()
     objid = id(var)
     if objid in context:
-        return func(name, "<...>")
+        return func(name, "<...>", **kwargs)
     context.add(objid)
     if isinstance(var, dict):
-        ret = func(name, dict((k, varmap(func, v, context, k)) for k, v in compat.iteritems(var)))
+        ret = func(name, dict((k, varmap(func, v, context, k, **kwargs)) for k, v in compat.iteritems(var)), **kwargs)
     elif isinstance(var, (list, tuple)):
-        ret = func(name, [varmap(func, f, context, name) for f in var])
+        ret = func(name, [varmap(func, f, context, name, **kwargs) for f in var], **kwargs)
     else:
-        ret = func(name, var)
+        ret = func(name, var, **kwargs)
     context.remove(objid)
     return ret
 

elasticapm/utils/encoding.py

@@ -179,7 +179,7 @@ def to_string(value):
         return to_unicode(value).encode("utf-8")
 
 
-def shorten(var, list_length=50, string_length=200, dict_length=50):
+def shorten(var, list_length=50, string_length=200, dict_length=50, **kwargs):
     """
     Shorten a given variable based on configurable maximum lengths, leaving
     breadcrumbs in the object to show that it was shortened.