adding regex for aws key and apikey values

Author: gabrielweiz

src/scrapy_settings_log/__init__.py

@@ -51,6 +51,15 @@ def spider_closed(self, spider):
         regex = settings.get("SETTINGS_LOGGING_REGEX")
         if regex is not None:
             settings = {k: v for k, v in settings.items() if re.search(regex, k)}
+        default_regexes = [
+            "(?i)(api[\W_]*key)",  # apikey and possible variations
+            "(?i)(AWS[\W_]*SECRET[\W_]*ACCESS[\W_]*KEY)"  # AWS_SECRET_ACCESS_KEY and possible variations
+        ]
+        if settings.get("MASKED_SENSITIVE_SETTINGS_ENABLED", True):
+            regex_list = settings.get("MASKED_SENSITIVE_SETTINGS_REGEX_LIST", default_regexes)
+            for reg in regex_list:
+                updated_settings = {k: '*'*len(v) for k, v in settings.items() if re.match(reg, k)}
+                settings = {**settings, **updated_settings}
 
         self.output_settings(settings, spider)
 

tests/test_code.py

@@ -103,3 +103,60 @@ class CustomClass:
         logger.spider_closed(spider)
 
     assert '{"DUMMY_CUSTOM_CLASS": {"CustomClass": "/foo/bar"}}' in caplog.text
+
+
+def test_log_all_should_not_return_apikey_value_by_default(caplog):
+    settings = {
+        "SETTINGS_LOGGING_ENABLED": True,
+        "APIKEY": 'apikey_value1',
+        "apikey": 'apikey_value2',
+        "api_key": 'apikey_value3',
+    }
+
+    spider = MockSpider(settings)
+    logger = SpiderSettingsLogging()
+    with caplog.at_level(logging.INFO):
+        logger.spider_closed(spider)
+
+    # won't check specifics here as the default settings
+    # can vary with scrapy versions - presence is enough
+    assert '"APIKEY": "*************"' in caplog.text
+    assert '"apikey": "*************"' in caplog.text
+    assert '"api_key": "*************"' in caplog.text
+    assert 'apikey_value' not in caplog.text
+
+
+def test_log_all_should_return_apikey_value_if_MASKED_SENSITIVE_SETTINGS_ENABLED_is_false(caplog):
+    settings = {
+        "SETTINGS_LOGGING_ENABLED": True,
+        "APIKEY": 'apikey_value',
+        "MASKED_SENSITIVE_SETTINGS_ENABLED": False,
+    }
+
+    spider = MockSpider(settings)
+    logger = SpiderSettingsLogging()
+    with caplog.at_level(logging.INFO):
+        logger.spider_closed(spider)
+
+    # won't check specifics here as the default settings
+    # can vary with scrapy versions - presence is enough
+    assert '"APIKEY": "apikey_value"' in caplog.text
+
+
+def test_log_all_should_not_return_aws_secret_key_value_by_default(caplog):
+    settings = {
+        "SETTINGS_LOGGING_ENABLED": True,
+        "AWS_SECRET_ACCESS_KEY": 'secret_value1',
+        "aws_secret_access_key": 'secret_value2',
+    }
+
+    spider = MockSpider(settings)
+    logger = SpiderSettingsLogging()
+    with caplog.at_level(logging.INFO):
+        logger.spider_closed(spider)
+
+    # won't check specifics here as the default settings
+    # can vary with scrapy versions - presence is enough
+    assert '"AWS_SECRET_ACCESS_KEY": "*************"' in caplog.text
+    assert '"aws_secret_access_key": "*************"' in caplog.text
+    assert 'secret_value' not in caplog.text