feat(MET-29): Add sensitive data filtering

- Implement SensitiveDataFilter class with 23+ default sensitive fields
- Automatic filtering of passwords, tokens, API keys, credit cards, etc.
- Case-insensitive matching with partial key matching support
- Recursive filtering for nested dicts and lists
- Configurable custom sensitive fields via DJANGO_SONAR settings
- Apply filter to POST/GET data, headers, and session data
- Refactor core module structure (parsers, collectors, filters)
- Add comprehensive test coverage (13+ tests for filtering)
- Update README with security documentation

Resolves: MET-29

Author: metalogico

.gitignore

@@ -159,4 +159,4 @@ cython_debug/
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 .idea/
 .vscode/
-.DS_Store
+.DS_Store

README.md

@@ -34,6 +34,7 @@ If you use this project, please consider giving it a ⭐.
   - Session vars
   - Headers
   - ...
+- 🔒 Automatic sensitive data filtering (passwords, tokens, API keys, etc.)
 - Historical data (clearable)
 - Simple and reactive UI
 
@@ -83,6 +84,35 @@ DJANGO_SONAR = {
 In this example I'm excluding all the http requests to static files, uploads, the sonar dashboard itself, the django admin panels and the browser reload library.
 Update this setting accordingly, YMMW.
 
+### 🔒 Sensitive Data Protection
+
+DjangoSonar automatically filters sensitive data from request payloads, headers, and session data before storing them in the database. By default, it masks common sensitive fields like:
+- `password`, `passwd`, `pwd`, `pass`
+- `token`, `api_key`, `secret`, `authorization`
+- `credit_card`, `cvv`, `ssn`, `pin`
+- And more...
+
+These fields are replaced with `***FILTERED***` in the stored data.
+
+**Custom Sensitive Fields**: You can add your own sensitive field patterns by adding them to the configuration:
+
+```python
+DJANGO_SONAR = {
+    'excludes': [
+        STATIC_URL,
+        MEDIA_URL,
+        '/sonar/',
+    ],
+    'sensitive_fields': [
+        'custom_secret',
+        'internal_api_key',
+        'private_data',
+    ],
+}
+```
+
+The filtering is case-insensitive and works with partial matches (e.g., `user_password`, `my_api_key` will also be filtered).
+
 5. Now you should be able to execute the migrations to create the two tables that DjangoSonar will use to collect the data.
 
 ```bash

django_sonar/core/__init__.py

@@ -0,0 +1,16 @@
+"""
+Django Sonar Core Module
+
+Core business logic for request processing, data collection, and filtering.
+"""
+
+from .parsers import RequestParser
+from .collectors import DataCollector
+from .filters import PathFilter, SensitiveDataFilter
+
+__all__ = [
+    'RequestParser',
+    'DataCollector',
+    'PathFilter',
+    'SensitiveDataFilter',
+]

django_sonar/core/collectors.py

@@ -0,0 +1,140 @@
+"""
+Data collection and persistence utilities.
+
+Handles saving request data to SonarData model with different categories:
+- details (user info, view function, memory usage)
+- payload (GET/POST data)
+- queries (database queries)
+- headers (request headers)
+- session (session data)
+- dumps (sonar() dumps)
+- exceptions (exception data)
+"""
+
+from django_sonar.models import SonarData
+from django_sonar import utils
+
+
+class DataCollector:
+    """Handles collection and persistence of request data"""
+
+    def __init__(self, sonar_request_uuid):
+        """
+        Initialize collector with request UUID.
+        
+        :param sonar_request_uuid: UUID of the SonarRequest instance
+        """
+        self.sonar_request_uuid = sonar_request_uuid
+
+    def save_details(self, user_info, view_func, middlewares_used, memory_diff):
+        """
+        Save request details (user, view, memory, middlewares).
+        
+        :param user_info: Dictionary with user information or None
+        :param view_func: String identifying the view function
+        :param middlewares_used: List/tuple of middleware names
+        :param memory_diff: Memory usage in MB
+        """
+        details = {
+            'user_info': user_info,
+            'view_func': view_func,
+            'middlewares_used': middlewares_used,
+            'memory_used': memory_diff
+        }
+        SonarData.objects.create(
+            sonar_request_id=self.sonar_request_uuid,
+            category='details',
+            data=details
+        )
+
+    def save_payload(self, get_payload, post_payload):
+        """
+        Save request payload (GET and POST/body data).
+        
+        :param get_payload: Dictionary with GET parameters
+        :param post_payload: Dictionary with POST/body data
+        """
+        payload = {
+            'get_payload': get_payload,
+            'post_payload': post_payload
+        }
+        SonarData.objects.create(
+            sonar_request_id=self.sonar_request_uuid,
+            category='payload',
+            data=payload
+        )
+
+    def save_queries(self, executed_queries):
+        """
+        Save database queries executed during request.
+        
+        :param executed_queries: List of query dictionaries from Django
+        """
+        queries = {
+            'executed_queries': executed_queries
+        }
+        SonarData.objects.create(
+            sonar_request_id=self.sonar_request_uuid,
+            category='queries',
+            data=queries
+        )
+
+    def save_headers(self, request_headers):
+        """
+        Save request headers.
+        
+        :param request_headers: Dictionary with request headers
+        """
+        headers = {
+            'request_headers': request_headers
+        }
+        SonarData.objects.create(
+            sonar_request_id=self.sonar_request_uuid,
+            category='headers',
+            data=headers
+        )
+
+    def save_session(self, session_data):
+        """
+        Save session data.
+        
+        :param session_data: Dictionary with session data
+        """
+        session = {
+            'session_data': session_data
+        }
+        SonarData.objects.create(
+            sonar_request_id=self.sonar_request_uuid,
+            category='session',
+            data=session
+        )
+
+    def save_dumps(self):
+        """
+        Save sonar() dumps from thread local storage.
+        
+        Retrieves dumps from utils.get_sonar_dump() and resets them.
+        """
+        sonar_dumps = utils.get_sonar_dump()
+        for dump in sonar_dumps:
+            SonarData.objects.create(
+                sonar_request_id=self.sonar_request_uuid,
+                category='dumps',
+                data=dump
+            )
+        utils.reset_sonar_dump()
+
+    def save_exceptions(self):
+        """
+        Save exception data from thread local storage.
+        
+        Retrieves exceptions from utils.get_sonar_exceptions() and resets them.
+        """
+        sonar_exceptions = utils.get_sonar_exceptions()
+        for ex in sonar_exceptions:
+            SonarData.objects.create(
+                sonar_request_id=self.sonar_request_uuid,
+                category='exception',
+                data=ex
+            )
+        utils.reset_sonar_exceptions()

django_sonar/core/filters.py

@@ -0,0 +1,170 @@
+"""
+Path filtering utilities.
+
+Handles exclusion of specific paths from monitoring based on:
+- Literal string matching (e.g., '/admin/')
+- Regex patterns (e.g., 'r^/api/v[0-9]+/')
+
+Sensitive data filtering utilities.
+
+Handles masking of sensitive data in payloads, headers, and session data.
+"""
+
+import re
+from django.conf import settings
+
+
+class PathFilter:
+    """Handles path exclusion logic for the middleware"""
+
+    def __init__(self):
+        """Initialize filter by compiling exclusion patterns from settings"""
+        self.compiled_excludes = self._compile_excludes()
+
+    def _compile_excludes(self):
+        """
+        Compile the excludes list provided in the settings.
+        
+        Patterns starting with 'r' are treated as regex patterns,
+        all others are treated as literal strings for startswith matching.
+        
+        :return: List of tuples (pattern_type, pattern) where pattern_type 
+                 is 'regex' or 'literal'
+        """
+        excluded_paths = settings.DJANGO_SONAR.get('excludes', [])
+        compiled = []
+        
+        for pattern in excluded_paths:
+            if isinstance(pattern, str) and pattern.startswith('r'):
+                try:
+                    regex_pattern = pattern[1:]
+                    compiled.append(('regex', re.compile(regex_pattern)))
+                except re.error:
+                    # If regex compilation fails, treat as literal
+                    compiled.append(('literal', pattern))
+            else:
+                compiled.append(('literal', pattern))
+        
+        return compiled
+
+    def should_exclude(self, path):
+        """
+        Check if the path should be excluded based on configured patterns.
+        
+        :param path: Request path to check
+        :return: True if path should be excluded, False otherwise
+        """
+        for pattern_type, pattern in self.compiled_excludes:
+            if pattern_type == 'regex':
+                if pattern.match(path):
+                    return True
+            else:  # literal
+                if path.startswith(pattern):
+                    return True
+        return False
+
+
+class SensitiveDataFilter:
+    """Handles sensitive data masking in request data"""
+    
+    # Default sensitive field patterns (case-insensitive)
+    DEFAULT_SENSITIVE_FIELDS = [
+        'password',
+        'passwd', 
+        'pwd',
+        'pass',
+        'secret',
+        'api_key',
+        'apikey',
+        'api_secret',
+        'token',
+        'access_token',
+        'refresh_token',
+        'auth',
+        'authorization',
+        'credit_card',
+        'card_number',
+        'cvv',
+        'cvc',
+        'ssn',
+        'pin',
+        'session_id',
+        'csrf',
+        'private_key',
+    ]
+    
+    MASK_VALUE = '***FILTERED***'
+    
+    def __init__(self):
+        """Initialize filter with configured sensitive fields"""
+        sonar_settings = getattr(settings, 'DJANGO_SONAR', {})
+        custom_fields = sonar_settings.get('sensitive_fields', [])
+        
+        # Combine default and custom fields, convert to lowercase for case-insensitive matching
+        self.sensitive_fields = set(
+            field.lower() for field in (self.DEFAULT_SENSITIVE_FIELDS + custom_fields)
+        )
+    
+    def _is_sensitive_key(self, key):
+        """
+        Check if a key represents sensitive data.
+        
+        :param key: Field name to check
+        :return: True if key is sensitive, False otherwise
+        """
+        if not isinstance(key, str):
+            return False
+            
+        key_lower = key.lower()
+        
+        # Check for exact matches
+        if key_lower in self.sensitive_fields:
+            return True
+        
+        # Check if any sensitive field is contained in the key
+        for sensitive_field in self.sensitive_fields:
+            if sensitive_field in key_lower:
+                return True
+        
+        return False
+    
+    def filter_dict(self, data):
+        """
+        Recursively filter sensitive data from a dictionary.
+        
+        :param data: Dictionary to filter
+        :return: Filtered dictionary with sensitive values masked
+        """
+        if not isinstance(data, dict):
+            return data
+        
+        filtered = {}
+        for key, value in data.items():
+            if self._is_sensitive_key(key):
+                filtered[key] = self.MASK_VALUE
+            elif isinstance(value, dict):
+                filtered[key] = self.filter_dict(value)
+            elif isinstance(value, (list, tuple)):
+                filtered[key] = self._filter_list(value)
+            else:
+                filtered[key] = value
+        
+        return filtered
+    
+    def _filter_list(self, data):
+        """
+        Filter sensitive data from a list/tuple.
+        
+        :param data: List or tuple to filter
+        :return: Filtered list with sensitive values masked
+        """
+        filtered = []
+        for item in data:
+            if isinstance(item, dict):
+                filtered.append(self.filter_dict(item))
+            elif isinstance(item, (list, tuple)):
+                filtered.append(self._filter_list(item))
+            else:
+                filtered.append(item)
+        
+        return filtered if isinstance(data, list) else tuple(filtered)