[internal] Add rate limited logger (#822)

* [internal] Add rate limited logger

* Use DD_LOGGING_RATE_LIMIT env key

* Use ddtrace.internal.logger.get_logger everywhere needed

* fix english

* Add comment about 'logging.setLoggerClass()'

* Add internal logger tests

* work on logger tests

* finish up tests for internal logger

Author: brettlangdon

.circleci/config.yml

@@ -86,6 +86,20 @@ jobs:
             - tracer.results
       - *save_cache_step
 
+  internal:
+    docker:
+      - *test_runner
+    resource_class: *resource_class
+    steps:
+      - checkout
+      - *restore_cache_step
+      - run: tox -e '{py27,py34,py35,py36}-internal' --result-json /tmp/internal.results
+      - persist_to_workspace:
+          root: /tmp
+          paths:
+            - internal.results
+      - *save_cache_step
+
   opentracer:
     docker:
       - *test_runner
@@ -1042,6 +1056,9 @@ workflows:
       - integration:
           requires:
             - flake8
+      - internal:
+          requires:
+            - flake8
       - jinja2:
           requires:
             - flake8
@@ -1156,6 +1173,7 @@ workflows:
             - grpc
             - httplib
             - integration
+            - internal
             - jinja2
             - kombu
             - mako

ddtrace/api.py

@@ -1,16 +1,16 @@
 # stdlib
-import logging
 import time
 import ddtrace
 from json import loads
 
 # project
 from .encoding import get_encoder, JSONEncoder
 from .compat import httplib, PYTHON_VERSION, PYTHON_INTERPRETER, get_connection_response
+from .internal.logger import get_logger
 from .utils.deprecation import deprecated
 
 
-log = logging.getLogger(__name__)
+log = get_logger(__name__)
 
 TRACE_COUNT_HEADER = 'X-Datadog-Trace-Count'
 

ddtrace/bootstrap/sitecustomize.py

@@ -9,6 +9,7 @@
 import logging
 
 from ddtrace.utils.formats import asbool, get_env
+from ddtrace.internal.logger import get_logger
 
 logs_injection = asbool(get_env('logs', 'injection'))
 DD_LOG_FORMAT = '%(asctime)s %(levelname)s [%(name)s] [%(filename)s:%(lineno)d] {}- %(message)s'.format(
@@ -32,7 +33,7 @@
 else:
     logging.basicConfig(format=DD_LOG_FORMAT)
 
-log = logging.getLogger(__name__)
+log = get_logger(__name__)
 
 EXTRA_PATCHED_MODULES = {
     "bottle": True,

ddtrace/commands/ddtrace_run.py

@@ -10,6 +10,11 @@
 if debug and debug.lower() == "true":
     logging.basicConfig(level=logging.DEBUG)
 
+# Do not use `ddtrace.internal.logger.get_logger` here
+# DEV: It isn't really necessary to use `DDLogger` here so we want to
+#        defer importing `ddtrace` until we actually need it.
+#      As well, no actual rate limiting would apply here since we only
+#        have a few logged lines
 log = logging.getLogger(__name__)
 
 USAGE = """

ddtrace/context.py

@@ -1,10 +1,10 @@
-import logging
 import threading
 
 from .constants import SAMPLING_PRIORITY_KEY, ORIGIN_KEY
+from .internal.logger import get_logger
 from .utils.formats import asbool, get_env
 
-log = logging.getLogger(__name__)
+log = get_logger(__name__)
 
 
 class Context(object):

ddtrace/contrib/cassandra/session.py

@@ -2,20 +2,20 @@
 Trace queries along a session to a cassandra cluster
 """
 import sys
-import logging
+
 # 3p
 import cassandra.cluster
 from ddtrace.vendor import wrapt
 
 # project
-from ddtrace import Pin
-from ddtrace.compat import stringify
-
-from ...utils.formats import deep_getattr
-from ...utils.deprecation import deprecated
+from ...compat import stringify
 from ...ext import net, cassandra as cassx, errors
+from ...internal.logger import get_logger
+from ...pin import Pin
+from ...utils.deprecation import deprecated
+from ...utils.formats import deep_getattr
 
-log = logging.getLogger(__name__)
+log = get_logger(__name__)
 
 RESOURCE_MAX_LENGTH = 5000
 SERVICE = 'cassandra'

ddtrace/contrib/celery/signals.py

@@ -1,13 +1,12 @@
-import logging
-
 from ddtrace import Pin, config
 
 from celery import registry
 
+from ...internal.logger import get_logger
 from . import constants as c
 from .utils import tags_from_context, retrieve_task_id, attach_span, detach_span, retrieve_span
 
-log = logging.getLogger(__name__)
+log = get_logger(__name__)
 SPAN_TYPE = 'worker'
 
 

ddtrace/contrib/dbapi/__init__.py

@@ -1,17 +1,14 @@
 """
 Generic dbapi tracing code.
 """
-
-import logging
-
-from ddtrace.vendor import wrapt
-
-from ddtrace import Pin
-from ddtrace.ext import AppTypes, sql
-from ddtrace.settings import config
-from ddtrace.utils.formats import asbool, get_env
-
-log = logging.getLogger(__name__)
+from ...ext import AppTypes, sql
+from ...internal.logger import get_logger
+from ...pin import Pin
+from ...settings import config
+from ...utils.formats import asbool, get_env
+from ...vendor import wrapt
+
+log = get_logger(__name__)
 
 config._add('dbapi2', dict(
     trace_fetch_methods=asbool(get_env('dbapi2', 'trace_fetch_methods', 'false')),

ddtrace/contrib/django/apps.py

@@ -1,5 +1,3 @@
-import logging
-
 # 3rd party
 from django.apps import AppConfig, apps
 
@@ -10,7 +8,9 @@
 from .templates import patch_template
 from .middleware import insert_exception_middleware, insert_trace_middleware
 
-log = logging.getLogger(__name__)
+from ...internal.logger import get_logger
+
+log = get_logger(__name__)
 
 
 class TracerConfig(AppConfig):

ddtrace/contrib/django/cache.py

@@ -1,14 +1,13 @@
-import logging
-
 from functools import wraps
 
 from django.conf import settings as django_settings
 
+from ...internal.logger import get_logger
 from .conf import settings, import_from_string
 from .utils import quantize_key_values, _resource_from_cache_prefix
 
 
-log = logging.getLogger(__name__)
+log = get_logger(__name__)
 
 # code instrumentation
 DATADOG_NAMESPACE = '__datadog_original_{method}'