Merge pull request #371 from DataDog/benjamin/distributed-tornado

Implement distributed tracing for Tornado web

Author: LotharSee

ddtrace/contrib/tornado/__init__.py

@@ -54,6 +54,7 @@ def notify(self):
         'datadog_trace': {
             'default_service': 'my-tornado-app',
             'tags': {'env': 'production'},
+            'distributed_tracing': True,
         },
     }
 
@@ -66,8 +67,11 @@ def notify(self):
 * ``default_service`` (default: `tornado-web`): set the service name used by the tracer. Usually
   this configuration must be updated with a meaningful name.
 * ``tags`` (default: `{}`): set global tags that should be applied to all spans.
-* ``enabled`` (default: `true`): define if the tracer is enabled or not. If set to `false`, the
+* ``enabled`` (default: `True`): define if the tracer is enabled or not. If set to `false`, the
   code is still instrumented but no spans are sent to the APM agent.
+* ``distributed_tracing`` (default: `False`): enable distributed tracing if this is called
+remotely from an instrumented application.
+We suggest to enable it only for internal services where headers are under your control.
 * ``agent_hostname`` (default: `localhost`): define the hostname of the APM agent.
 * ``agent_port`` (default: `8126`): define the port of the APM agent.
 """

ddtrace/contrib/tornado/application.py

@@ -20,6 +20,7 @@ def tracer_config(__init__, app, args, kwargs):
     settings = {
         'tracer': ddtrace.tracer,
         'default_service': 'tornado-web',
+        'distributed_tracing': False,
     }
 
     # update defaults with users settings

ddtrace/contrib/tornado/handlers.py

@@ -3,6 +3,7 @@
 from .constants import CONFIG_KEY, REQUEST_CONTEXT_KEY, REQUEST_SPAN_KEY
 from .stack_context import TracerStackContext
 from ...ext import http
+from ...propagation.http import HTTPPropagator
 
 
 def execute(func, handler, args, kwargs):
@@ -15,11 +16,19 @@ def execute(func, handler, args, kwargs):
     settings = handler.settings[CONFIG_KEY]
     tracer = settings['tracer']
     service = settings['default_service']
+    distributed_tracing = settings['distributed_tracing']
 
     with TracerStackContext():
         # attach the context to the request
         setattr(handler.request, REQUEST_CONTEXT_KEY, tracer.get_call_context())
 
+        # Read and use propagated context from HTTP headers
+        if distributed_tracing:
+            propagator = HTTPPropagator()
+            context = propagator.extract(handler.request.headers)
+            if context.trace_id:
+                tracer.context_provider.activate(context)
+
         # store the request span in the request so that it can be used later
         request_span = tracer.trace(
             'tornado.request',

docs/index.rst

@@ -286,15 +286,21 @@ Supported web frameworks:
 
 - Django
 - Flask
+- Tornado
 
+For web servers not supported, you can extract the HTTP context from the headers using the `HTTPPropagator`.
 
-HTTP client/server
-~~~~~~~~~~~~~~~~~~
+.. autoclass:: ddtrace.propagation.http.HTTPPropagator
+    :members: extract
+
+HTTP client
+~~~~~~~~~~~
 
-You can use the `HTTPPropagator` manually when used with an HTTP client or if your web framework isn't supported.
+When calling a remote HTTP server part of the distributed trace, you have to propagate the HTTP headers.
+This is not done automatically to prevent your system from leaking tracing information to external services.
 
 .. autoclass:: ddtrace.propagation.http.HTTPPropagator
-    :members:
+    :members: inject
 
 Custom
 ~~~~~~

tests/contrib/tornado/test_tornado_web.py

@@ -3,11 +3,20 @@
 from .web.app import CustomDefaultHandler
 from .utils import TornadoTestCase
 
+from ddtrace.constants import SAMPLING_PRIORITY_KEY
+
 
 class TestTornadoWeb(TornadoTestCase):
     """
     Ensure that Tornado web handlers are properly traced.
     """
+    def get_settings(self):
+        return {
+            'datadog_trace': {
+                'distributed_tracing': True,
+            }
+        }
+
     def test_success_handler(self):
         # it should trace a handler that returns 200
         response = self.fetch('/success/')
@@ -227,6 +236,69 @@ def test_static_handler(self):
         eq_('/statics/empty.txt', request_span.get_tag('http.url'))
         eq_(0, request_span.error)
 
+    def test_propagation(self):
+        # it should trace a handler that returns 200 with a propagated context
+        headers = {
+            'x-datadog-trace-id': '1234',
+            'x-datadog-parent-id': '4567',
+            'x-datadog-sampling-priority': '2'
+        }
+        response = self.fetch('/success/', headers=headers)
+        eq_(200, response.code)
+
+        traces = self.tracer.writer.pop_traces()
+        eq_(1, len(traces))
+        eq_(1, len(traces[0]))
+
+        request_span = traces[0][0]
+
+        # simple sanity check on the span
+        eq_('tornado.request', request_span.name)
+        eq_('200', request_span.get_tag('http.status_code'))
+        eq_('/success/', request_span.get_tag('http.url'))
+        eq_(0, request_span.error)
+
+        # check propagation
+        eq_(1234, request_span.trace_id)
+        eq_(4567, request_span.parent_id)
+        eq_(2, request_span.get_metric(SAMPLING_PRIORITY_KEY))
+
+
+class TestNoPropagationTornadoWeb(TornadoTestCase):
+    """
+    Ensure that Tornado web handlers are properly traced and are ignoring propagated HTTP headers when disabled.
+    """
+    def get_settings(self):
+        # distributed_tracing should be disabled by default
+        return {}
+
+    def test_no_propagation(self):
+        # it should not propagate the HTTP context
+        headers = {
+            'x-datadog-trace-id': '1234',
+            'x-datadog-parent-id': '4567',
+            'x-datadog-sampling-priority': '2'
+        }
+        response = self.fetch('/success/', headers=headers)
+        eq_(200, response.code)
+
+        traces = self.tracer.writer.pop_traces()
+        eq_(1, len(traces))
+        eq_(1, len(traces[0]))
+
+        request_span = traces[0][0]
+
+        # simple sanity check on the span
+        eq_('tornado.request', request_span.name)
+        eq_('200', request_span.get_tag('http.status_code'))
+        eq_('/success/', request_span.get_tag('http.url'))
+        eq_(0, request_span.error)
+
+        # check non-propagation
+        assert request_span.trace_id != 1234
+        assert request_span.parent_id != 4567
+        assert request_span.get_metric(SAMPLING_PRIORITY_KEY) != 2
+
 
 class TestCustomTornadoWeb(TornadoTestCase):
     """