Adding custom sentry sanitizer

Author: AddisonSchiller

tests/server/test_sanitize.py

@@ -0,0 +1,76 @@
+import pytest
+from unittest import mock
+
+from waterbutler.server.sanitize import WBSanitizer
+
+
+@pytest.fixture
+def sanitizer():
+    return WBSanitizer(mock.Mock())
+
+
+class TestWBSanitizer:
+    # The sanitize function changes some strings and dictionaries
+    # you put into it, so you need to explicitly test most things
+
+    MASK = '*' * 8
+
+    def test_no_sanitization(self, sanitizer):
+        assert sanitizer.sanitize('thing', 'ghost science') == 'ghost science'
+
+    def test_fields_sanitized(self, sanitizer):
+        fields = sanitizer.FIELDS
+        for field in fields:
+            assert sanitizer.sanitize(field, 'free speech') == self.MASK
+
+    def test_value_is_none(self, sanitizer):
+        assert sanitizer.sanitize('great hair', None) is None
+
+    def test_sanitize_credit_card(self, sanitizer):
+        assert sanitizer.sanitize('credit', '424242424242424') == self.MASK
+        assert sanitizer.sanitize('credit', '4242424242424243333333') != self.MASK
+
+    def test_sanitize_dictionary(self, sanitizer):
+        value_dict = {
+            'great_entry': 'very much not a secret or credit card'
+        }
+
+        result = sanitizer.sanitize('value_dict', value_dict)
+        assert result == {
+            'great_entry': 'very much not a secret or credit card'
+        }
+
+        sanitize_dict = {
+            'key': 'secret',
+            'okay_value': 'bears are awesome'
+        }
+        result = result = sanitizer.sanitize('sanitize_dict', sanitize_dict)
+
+        # Sanity check
+        assert result != {
+            'key': 'secret',
+            'okay_value': 'bears are awesome'
+        }
+
+        assert result == {
+            'key': '*' * 8,
+            'okay_value': 'bears are awesome'
+        }
+
+    def test_dataverse_secret(self, sanitizer):
+
+        # Named oddly because if you call it `dv_secret` it will get sanitized by a different
+        # part of the sanitizer
+        dv_value = 'aaaaaaaa-bbbb-bbbb-bbbb-cccccccccccc'
+        assert sanitizer.sanitize('dv_value', dv_value) == self.MASK
+
+        dv_value = 'random characters and other things  aaaaaaaa-bbbb-bbbb-bbbb-cccccccccccc'
+        expected = 'random characters and other things  ' + self.MASK
+        assert sanitizer.sanitize('dv_value', dv_value) == expected
+
+    def test_bytes(self, sanitizer):
+        key = b'key'
+        assert sanitizer.sanitize(key, 'bossy yogurt') == self.MASK
+
+        other_key = b'should_be_safe'
+        assert sanitizer.sanitize(other_key, 'snow science') == 'snow science'

waterbutler/server/app.py

@@ -45,7 +45,8 @@ def make_app(debug):
         [(r'/status', handlers.StatusHandler)],
         debug=debug,
     )
-    app.sentry_client = AsyncSentryClient(settings.SENTRY_DSN, release=waterbutler.__version__)
+    app.sentry_client = AsyncSentryClient(settings.SENTRY_DSN, release=waterbutler.__version__,
+                                        processors=('waterbutler.server.sanitize.WBSanitizer',))
     return app
 
 

waterbutler/server/sanitize.py

@@ -0,0 +1,71 @@
+import re
+
+from raven.processors import SanitizePasswordsProcessor
+
+
+class WBSanitizer(SanitizePasswordsProcessor):
+    """Asterisk out things that look like passwords, keys, etc."""
+
+    # Store mask as a fixed length for security
+    MASK = '*' * 8
+
+    # Token and key added from original. Key is used by Dataverse
+    FIELDS = frozenset([
+        'password',
+        'secret',
+        'passwd',
+        'authorization',
+        'api_key',
+        'apikey',
+        'sentry_dsn',
+        'access_token',
+        'key',
+        'token',
+    ])
+
+    # Credit card regex left intact from original processor
+    # While we should never have credit card information, its still best to perform the check
+    # and keep old functionality
+    VALUES_RE = re.compile(r'^(?:\d[ -]*?){13,16}$')
+
+    # Should specifically match Dataverse secrets. Key format checked on demo and on Harvard
+    DATAVERSE_SECRET_RE = re.compile(r'[A-Za-z0-9]{8}-[A-Za-z0-9]{4}-[A-Za-z0-9]'
+                                                '{4}-[A-Za-z0-9]{4}-[A-Za-z0-9]{12}')
+
+    def sanitize(self, key, value):
+        """Overload the sanitize function of the `SanitizePasswordsProcessor'."""
+        if value is None:
+            return
+
+        # Part of the original method. Looks for credit cards to sanitize
+        if isinstance(value, str) and self.VALUES_RE.search(value):
+            return self.MASK
+
+        if isinstance(value, dict):
+            for item in value:
+                if item in self.FIELDS:
+                    value[item] = self.MASK
+
+        # Check for Dataverse secrets
+        if isinstance(value, str):
+            matches = self.DATAVERSE_SECRET_RE.findall(value)
+            for match in matches:
+                value = value.replace(match, self.MASK)
+
+        # key can be a NoneType
+        if not key:
+            return value
+
+        # Just in case we have bytes here, we want to turn them into text
+        # properly without failing so we can perform our check.
+        if isinstance(key, bytes):
+            key = key.decode('utf-8', 'replace')
+        else:
+            key = str(key)
+
+        key = key.lower()
+        for field in self.FIELDS:
+            if field in key:
+                return self.MASK
+
+        return value