Merge pull request #704 from DataDog/0.16-dev

v0.16.0

Author: brettlangdon

.circleci/config.yml

@@ -143,12 +143,23 @@ jobs:
       TOX_SKIP_DIST: False
     steps:
       - checkout
-      - *restore_cache_step
       - run: tox -e '{py27,py34,py35,py36}-ddtracerun' --result-json /tmp/ddtracerun.results
       - persist_to_workspace:
           root: /tmp
           paths:
             - ddtracerun.results
+
+  test_utils:
+    docker:
+      - *test_runner
+    steps:
+      - checkout
+      - *restore_cache_step
+      - run: tox -e '{py27,py34,py35,py36}-test_utils' --result-json /tmp/test_utils.results
+      - persist_to_workspace:
+          root: /tmp
+          paths:
+            - test_utils.results
       - *save_cache_step
 
   asyncio:
@@ -213,13 +224,11 @@ jobs:
     steps:
       - checkout
       - *restore_cache_step
-      - run: tox -e 'bottle_contrib-{py27,py34,py35,py36}-bottle{11,12}-webtest' --result-json /tmp/bottle.1.results
-      - run: TOX_SKIP_DIST=False tox -e 'bottle_contrib_autopatch-{py27,py34,py35,py36}-bottle{11,12}-webtest' --result-json /tmp/bottle.2.results
+      - run: tox -e 'bottle_contrib{,_autopatch}-{py27,py34,py35,py36}-bottle{11,12}-webtest' --result-json /tmp/bottle.results
       - persist_to_workspace:
           root: /tmp
           paths:
-            - bottle.1.results
-            - bottle.2.results
+            - bottle.results
       - *save_cache_step
 
   cassandra:
@@ -245,12 +254,14 @@ jobs:
 
   celery:
     docker:
-      - *test_runner
+      - <<: *test_runner
+        env:
+          TOX_SKIP_DIST: False
       - image: redis:3.2-alpine
     steps:
       - checkout
       - *restore_cache_step
-      - run: TOX_SKIP_DIST=False tox -e 'celery_contrib-{py27,py34,py35,py36}-celery{31,40,41,42}-redis{210}' --result-json /tmp/celery.results
+      - run: tox -e 'celery_contrib-{py27,py34,py35,py36}-celery{31,40,41,42}-redis{210}' --result-json /tmp/celery.results
       - persist_to_workspace:
           root: /tmp
           paths:
@@ -264,7 +275,7 @@ jobs:
     steps:
       - checkout
       - *restore_cache_step
-      - run: TOX_SKIP_DIST=False tox -e 'elasticsearch_contrib-{py27,py34,py35,py36}-elasticsearch{16,17,18,23,24,51,52,53,54,63}' --result-json /tmp/elasticsearch.results
+      - run: tox -e 'elasticsearch_contrib-{py27,py34,py35,py36}-elasticsearch{16,17,18,23,24,51,52,53,54,63}' --result-json /tmp/elasticsearch.results
       - persist_to_workspace:
           root: /tmp
           paths:
@@ -277,13 +288,11 @@ jobs:
     steps:
       - checkout
       - *restore_cache_step
-      - run: tox -e 'falcon_contrib-{py27,py34,py35,py36}-falcon{10,11,12,13,14}' --result-json /tmp/falcon.1.results
-      - run: TOX_SKIP_DIST=False tox -e 'falcon_contrib_autopatch-{py27,py34,py35,py36}-falcon{10,11,12,13,14}' --result-json /tmp/falcon.2.results
+      - run: tox -e 'falcon_contrib{,_autopatch}-{py27,py34,py35,py36}-falcon{10,11,12,13,14}' --result-json /tmp/falcon.results
       - persist_to_workspace:
           root: /tmp
           paths:
-            - falcon.1.results
-            - falcon.2.results
+            - falcon.results
       - *save_cache_step
 
   django:
@@ -299,21 +308,17 @@ jobs:
     steps:
       - checkout
       - *restore_cache_step
-      - run: tox -e 'django_contrib-{py27,py34,py35,py36}-django{18,111}-djangopylibmc06-djangoredis45-pylibmc-redis{210}-memcached' --result-json /tmp/django.1.results
-      - run: TOX_SKIP_DIST=False tox -e 'django_contrib_autopatch-{py27,py34,py35,py36}-django{18,111}-djangopylibmc06-djangoredis45-pylibmc-redis{210}-memcached' --result-json /tmp/django.2.results
-      - run: tox -e 'django_drf_contrib-{py27,py34,py35,py36}-django{111}-djangorestframework{34,37,38}' --result-json /tmp/django.3.results
-      - run: tox -e 'django_contrib-{py34,py35,py36}-django{200}-djangopylibmc06-djangoredis45-pylibmc-redis{210}-memcached' --result-json /tmp/django.4.results
-      - run: TOX_SKIP_DIST=False tox -e 'django_contrib_autopatch-{py34,py35,py36}-django{200}-djangopylibmc06-djangoredis45-pylibmc-redis{210}-memcached' --result-json /tmp/django.5.results
-      - run: tox -e 'django_drf_contrib-{py34,py35,py36}-django{200}-djangorestframework{37,38}' --result-json /tmp/django.6.results
+      - run: tox -e 'django_contrib{,_autopatch}-{py27,py34,py35,py36}-django{18,111}-djangopylibmc06-djangoredis45-pylibmc-redis{210}-memcached' --result-json /tmp/django.1.results
+      - run: tox -e 'django_drf_contrib-{py27,py34,py35,py36}-django{111}-djangorestframework{34,37,38}' --result-json /tmp/django.2.results
+      - run: tox -e 'django_contrib{,_autopatch}-{py34,py35,py36}-django{200}-djangopylibmc06-djangoredis45-pylibmc-redis{210}-memcached' --result-json /tmp/django.3.results
+      - run: tox -e 'django_drf_contrib-{py34,py35,py36}-django{200}-djangorestframework{37,38}' --result-json /tmp/django.4.results
       - persist_to_workspace:
           root: /tmp
           paths:
             - django.1.results
             - django.2.results
             - django.3.results
             - django.4.results
-            - django.5.results
-            - django.6.results
       - *save_cache_step
 
   flask:
@@ -326,10 +331,12 @@ jobs:
       - *restore_cache_step
       - run: tox -e 'flask_contrib-{py27,py34,py35,py36}-flask{010,011,012,10}-blinker' --result-json /tmp/flask.1.results
       - run: TOX_SKIP_DIST=False tox -e 'flask_contrib_autopatch-{py27,py34,py35,py36}-flask{010,011,012,10}-blinker' --result-json /tmp/flask.2.results
-      - run: tox -e 'flask_cache_contrib-{py27,py34,py35,py36}-flask{010,011,012}-flaskcache{013}-memcached-redis{210}-blinker' --result-json /tmp/flask.3.results
-      - run: TOX_SKIP_DIST=False tox -e 'flask_cache_contrib_autopatch-{py27,py34,py35,py36}-flask{010,011,012}-flaskcache{013}-memcached-redis{210}-blinker' --result-json /tmp/flask.4.results
-      - run: tox -e 'flask_cache_contrib-{py27}-flask{010,011}-flaskcache{012}-memcached-redis{210}-blinker' --result-json /tmp/flask.5.results
-      - run: TOX_SKIP_DIST=False tox -e 'flask_cache_contrib_autopatch-{py27}-flask{010,011}-flaskcache{012}-memcached-redis{210}-blinker' --result-json /tmp/flask.6.results
+      - run: tox -e 'flask_contrib-{py27}-flask{09}-blinker' --result-json /tmp/flask.3.results
+      - run: TOX_SKIP_DIST=False tox -e 'flask_contrib_autopatch-{py27}-flask{09}-blinker' --result-json /tmp/flask.4.results
+      - run: tox -e 'flask_cache_contrib-{py27,py34,py35,py36}-flask{010,011,012}-flaskcache{013}-memcached-redis{210}-blinker' --result-json /tmp/flask.5.results
+      - run: TOX_SKIP_DIST=False tox -e 'flask_cache_contrib_autopatch-{py27,py34,py35,py36}-flask{010,011,012}-flaskcache{013}-memcached-redis{210}-blinker' --result-json /tmp/flask.6.results
+      - run: tox -e 'flask_cache_contrib-{py27}-flask{010,011}-flaskcache{012}-memcached-redis{210}-blinker' --result-json /tmp/flask.7.results
+      - run: TOX_SKIP_DIST=False tox -e 'flask_cache_contrib_autopatch-{py27}-flask{010,011}-flaskcache{012}-memcached-redis{210}-blinker' --result-json /tmp/flask.8.results
       - persist_to_workspace:
           root: /tmp
           paths:
@@ -339,6 +346,8 @@ jobs:
             - flask.4.results
             - flask.5.results
             - flask.6.results
+            - flask.7.results
+            - flask.8.results
       - *save_cache_step
 
   gevent:
@@ -347,7 +356,7 @@ jobs:
     steps:
       - checkout
       - *restore_cache_step
-      - run: tox -e 'gevent_contrib-{py27,py34,py35,py36}-gevent{11,12}' --result-json /tmp/gevent.1.results
+      - run: tox -e 'gevent_contrib-{py27,py34,py35,py36}-gevent{11,12,13}' --result-json /tmp/gevent.1.results
       - run: tox -e 'gevent_contrib-{py27}-gevent{10}' --result-json /tmp/gevent.2.results
       - persist_to_workspace:
           root: /tmp
@@ -369,6 +378,19 @@ jobs:
             - httplib.results
       - *save_cache_step
 
+  grpc:
+    docker:
+      - *test_runner
+    steps:
+      - checkout
+      - *restore_cache_step
+      - run: tox -e 'grpc_contrib-{py27,py34,py35,py36}-grpc' --result-json /tmp/grpc.results
+      - persist_to_workspace:
+          root: /tmp
+          paths:
+            - grpc.results
+      - *save_cache_step
+
   mysqlconnector:
     docker:
       - *test_runner
@@ -470,13 +492,11 @@ jobs:
     steps:
       - checkout
       - *restore_cache_step
-      - run: tox -e 'pymemcache_contrib-{py27,py34,py35,py36}-pymemcache{130,140}' --result-json /tmp/pymemcache.1.results
-      - run: TOX_SKIP_DIST=False tox -e 'pymemcache_contrib_autopatch-{py27,py34,py35,py36}-pymemcache{130,140}' --result-json /tmp/pymemcache.2.results
+      - run: tox -e 'pymemcache_contrib{,_autopatch}-{py27,py34,py35,py36}-pymemcache{130,140}' --result-json /tmp/pymemcache.results
       - persist_to_workspace:
           root: /tmp
           paths:
-            - pymemcache.1.results
-            - pymemcache.2.results
+            - pymemcache.results
       - *save_cache_step
 
   mongoengine:
@@ -513,13 +533,11 @@ jobs:
     steps:
       - checkout
       - *restore_cache_step
-      - run: tox -e 'pyramid_contrib-{py27,py34,py35,py36}-pyramid{17,18,19}-webtest' --result-json /tmp/pyramid.1.results
-      - run: TOX_SKIP_DIST=False tox -e 'pyramid_contrib_autopatch-{py27,py34,py35,py36}-pyramid{17,18,19}-webtest' --result-json /tmp/pyramid.2.results
+      - run: tox -e 'pyramid_contrib{,_autopatch}-{py27,py34,py35,py36}-pyramid{17,18,19}-webtest' --result-json /tmp/pyramid.results
       - persist_to_workspace:
           root: /tmp
           paths:
-            - pyramid.1.results
-            - pyramid.2.results
+            - pyramid.results
       - *save_cache_step
 
   requests:
@@ -529,7 +547,7 @@ jobs:
     steps:
       - checkout
       - *restore_cache_step
-      - run: tox -e 'requests_contrib-{py27,py34,py35,py36}-requests{208,209,210,211,212,213,219}' --result-json /tmp/requests.results
+      - run: tox -e 'requests_contrib{,_autopatch}-{py27,py34,py35,py36}-requests{208,209,210,211,212,213,219}' --result-json /tmp/requests.results
       - persist_to_workspace:
           root: /tmp
           paths:
@@ -677,6 +695,21 @@ jobs:
             - vertica.results
       - *save_cache_step
 
+  kombu:
+    docker:
+    - *test_runner
+    - image: rabbitmq:3.7-alpine
+    steps:
+    - checkout
+    - *restore_cache_step
+    - run: tox -e wait rabbitmq
+    - run: tox -e 'kombu_contrib-{py27,py34,py35,py36}-kombu{40,41,42}' --result-json /tmp/kombu.results
+    - persist_to_workspace:
+        root: /tmp
+        paths:
+        - kombu.results
+    - *save_cache_step
+
   sqlite3:
     docker:
       - *test_runner
@@ -724,6 +757,19 @@ jobs:
       - run: sudo pip install mkwheelhouse sphinx awscli wrapt
       - run: VERSION_SUFFIX=$CIRCLE_BRANCH$CIRCLE_BUILD_NUM S3_DIR=trace-dev rake release:wheel
 
+  jinja2:
+    docker:
+      - *test_runner
+    steps:
+      - checkout
+      - *restore_cache_step
+      - run: tox -e 'jinja2_contrib-{py27,py34,py35,py36}-jinja{27,28,29,210}' --result-json /tmp/jinja2.results
+      - persist_to_workspace:
+          root: /tmp
+          paths:
+            - jinja2.results
+      - *save_cache_step
+
   build_docs:
     # deploy official documentation
     docker:
@@ -797,6 +843,7 @@ workflows:
       - futures
       - boto
       - ddtracerun
+      - test_utils
       - asyncio
       - pylons
       - aiohttp
@@ -810,6 +857,7 @@ workflows:
       - flask
       - gevent
       - httplib
+      - grpc
       - mongoengine
       - mysqlconnector
       - mysqlpython
@@ -827,9 +875,11 @@ workflows:
       - aiopg
       - redis
       - rediscluster
+      - kombu
       - sqlite3
       - msgpack
       - vertica
+      - jinja2
       - build_docs
       - wait_all_tests:
           requires:
@@ -840,6 +890,7 @@ workflows:
             - futures
             - boto
             - ddtracerun
+            - test_utils
             - asyncio
             - pylons
             - aiohttp
@@ -852,6 +903,7 @@ workflows:
             - django
             - flask
             - gevent
+            - grpc
             - httplib
             - mongoengine
             - mysqlconnector
@@ -870,9 +922,11 @@ workflows:
             - aiopg
             - redis
             - rediscluster
+            - kombu
             - sqlite3
             - msgpack
             - vertica
+            - jinja2
             - build_docs
       - deploy_dev:
           requires:

ddtrace/__init__.py

@@ -2,13 +2,12 @@
 from .pin import Pin
 from .span import Span
 from .tracer import Tracer
-from .settings import Config
+from .settings import config
 
-__version__ = '0.15.0'
+__version__ = '0.16.0'
 
 # a global tracer instance with integration settings
 tracer = Tracer()
-config = Config()
 
 __all__ = [
     'patch',

ddtrace/compat.py

@@ -1,5 +1,5 @@
-import sys
 import platform
+import sys
 
 PYTHON_VERSION_INFO = sys.version_info
 PY2 = sys.version_info[0] == 2

ddtrace/context.py

@@ -2,7 +2,7 @@
 import threading
 
 from .constants import SAMPLING_PRIORITY_KEY
-
+from .utils.formats import asbool, get_env
 
 log = logging.getLogger(__name__)
 
@@ -22,6 +22,9 @@ class Context(object):
 
     This data structure is thread-safe.
     """
+    _partial_flush_enabled = asbool(get_env('tracer', 'partial_flush_enabled', 'false'))
+    _partial_flush_min_spans = int(get_env('tracer', 'partial_flush_min_spans', 500))
+
     def __init__(self, trace_id=None, span_id=None, sampled=True, sampling_priority=None):
         """
         Initialize a new thread-safe ``Context``.
@@ -190,6 +193,31 @@ def get(self):
                 self._sampling_priority = None
                 self._sampled = True
                 return trace, sampled
+
+            elif self._partial_flush_enabled and self._finished_spans >= self._partial_flush_min_spans:
+                # partial flush when enabled and we have more than the minimal required spans
+                trace = self._trace
+                sampled = self._sampled
+                sampling_priority = self._sampling_priority
+                # attach the sampling priority to the context root span
+                if sampled and sampling_priority is not None and trace:
+                    trace[0].set_metric(SAMPLING_PRIORITY_KEY, sampling_priority)
+
+                # Any open spans will remain as `self._trace`
+                # Any finished spans will get returned to be flushed
+                opened_spans = []
+                closed_spans = []
+                for span in trace:
+                    if span._finished:
+                        closed_spans.append(span)
+                    else:
+                        opened_spans.append(span)
+
+                # Update trace spans and stats
+                self._trace = opened_spans
+                self._finished_spans = 0
+
+                return closed_spans, sampled
             else:
                 return None, None
 

ddtrace/contrib/bottle/trace.py

@@ -8,6 +8,8 @@
 # project
 from ...propagation.http import HTTPPropagator
 
+SPAN_TYPE = 'web'
+
 class TracePlugin(object):
     name = 'trace'
     api = 2
@@ -37,7 +39,7 @@ def wrapped(*args, **kwargs):
                 if context.trace_id:
                     self.tracer.context_provider.activate(context)
 
-            with self.tracer.trace('bottle.request', service=self.service, resource=resource) as s:
+            with self.tracer.trace('bottle.request', service=self.service, resource=resource, span_type=SPAN_TYPE) as s:
                 code = 0
                 try:
                     return callback(*args, **kwargs)