[core] Add API to configure Trace Search (#781)

* Add API to configure Trace Search, enable it for Pylons

* fix up styles

* allow asserting on span metrics

* all integrations have a default 'event_sample_rate' option

* [core] set event sample rate key for flask.request spans

* remove unnecessary Django test

* fix root span comparison

* [core] set event sample rate key for aiohttp.request spans

* ensure all bottle tests are running

* [core] set event sample rate key for bottle.request spans

* work on other integrations and django tests

* update tests for pylons

* update falcon tests

* fix pyramid tests

* assert that we have a root span

* fix molten tests

* fix tornado tests

* fix broken tests

Author: brettlangdon

ddtrace/constants.py

@@ -1,3 +1,6 @@
 FILTERS_KEY = 'FILTERS'
-SAMPLE_RATE_METRIC_KEY = "_sample_rate"
+SAMPLE_RATE_METRIC_KEY = '_sample_rate'
 SAMPLING_PRIORITY_KEY = '_sampling_priority_v1'
+EVENT_SAMPLE_RATE_KEY = '_dd1.sr.eausr'
+
+NUMERIC_TAGS = (EVENT_SAMPLE_RATE_KEY, )

ddtrace/contrib/aiohttp/middlewares.py

@@ -1,10 +1,11 @@
 import asyncio
 
 from ..asyncio import context_provider
-from ...ext import AppTypes, http
 from ...compat import stringify
-from ...context import Context
+from ...constants import EVENT_SAMPLE_RATE_KEY
+from ...ext import AppTypes, http
 from ...propagation.http import HTTPPropagator
+from ...settings import config
 
 
 CONFIG_KEY = 'datadog_trace'
@@ -45,6 +46,10 @@ def attach_context(request):
             span_type=http.TYPE,
         )
 
+        # Configure trace search sample rate
+        if config.aiohttp.event_sample_rate is not None:
+            request_span.set_tag(EVENT_SAMPLE_RATE_KEY, config.aiohttp.event_sample_rate)
+
         # attach the context and the root span to the request; the Context
         # may be freely used by the application code
         request[REQUEST_CONTEXT_KEY] = request_span.context

ddtrace/contrib/bottle/trace.py

@@ -6,7 +6,9 @@
 from ddtrace.ext import http, AppTypes
 
 # project
+from ...constants import EVENT_SAMPLE_RATE_KEY
 from ...propagation.http import HTTPPropagator
+from ...settings import config
 
 SPAN_TYPE = 'web'
 
@@ -41,6 +43,10 @@ def wrapped(*args, **kwargs):
                     self.tracer.context_provider.activate(context)
 
             with self.tracer.trace('bottle.request', service=self.service, resource=resource, span_type=SPAN_TYPE) as s:
+                # Configure trace search sample rate
+                if config.bottle.event_sample_rate is not None:
+                    s.set_tag(EVENT_SAMPLE_RATE_KEY, config.bottle.event_sample_rate)
+
                 code = 0
                 try:
                     return callback(*args, **kwargs)

ddtrace/contrib/django/middleware.py

@@ -4,9 +4,11 @@
 from .conf import settings
 from .compat import user_is_authenticated, get_resolver
 
-from ...ext import http
+from ...constants import EVENT_SAMPLE_RATE_KEY
 from ...contrib import func_name
+from ...ext import http
 from ...propagation.http import HTTPPropagator
+from ...settings import config
 
 # 3p
 from django.core.exceptions import MiddlewareNotUsed
@@ -118,6 +120,10 @@ def process_request(self, request):
                 span_type=http.TYPE,
             )
 
+            # Configure trace search sample rate
+            if config.django.event_sample_rate is not None:
+                span.set_tag(EVENT_SAMPLE_RATE_KEY, config.django.event_sample_rate)
+
             span.set_tag(http.METHOD, request.method)
             span.set_tag(http.URL, request.path)
             _set_req_span(request, span)

ddtrace/contrib/falcon/middleware.py

@@ -3,7 +3,9 @@
 from ddtrace.ext import http as httpx
 from ddtrace.http import store_request_headers, store_response_headers
 from ddtrace.propagation.http import HTTPPropagator
+
 from ...compat import iteritems
+from ...constants import EVENT_SAMPLE_RATE_KEY
 from ...ext import AppTypes
 from ...settings import config
 
@@ -37,6 +39,10 @@ def process_request(self, req, resp):
             span_type=httpx.TYPE,
         )
 
+        # Configure trace search sample rate
+        if config.falcon.event_sample_rate is not None:
+            span.set_tag(EVENT_SAMPLE_RATE_KEY, config.falcon.event_sample_rate)
+
         span.set_tag(httpx.METHOD, req.method)
         span.set_tag(httpx.URL, req.url)
 

ddtrace/contrib/flask/patch.py

@@ -7,6 +7,7 @@
 
 from ddtrace import config, Pin
 
+from ...constants import EVENT_SAMPLE_RATE_KEY
 from ...ext import AppTypes
 from ...ext import http
 from ...propagation.http import HTTPPropagator
@@ -284,6 +285,10 @@ def traced_wsgi_app(pin, wrapped, instance, args, kwargs):
     # We will override this below in `traced_dispatch_request` when we have a `RequestContext` and possibly a url rule
     resource = u'{} {}'.format(request.method, request.path)
     with pin.tracer.trace('flask.request', service=pin.service, resource=resource, span_type=http.TYPE) as s:
+        # Configure trace search sample rate
+        if config.flask.event_sample_rate is not None:
+            s.set_tag(EVENT_SAMPLE_RATE_KEY, config.flask.event_sample_rate)
+
         s.set_tag(FLASK_VERSION, flask_version_str)
 
         # Wrap the `start_response` handler to extract response code

ddtrace/contrib/molten/patch.py

@@ -4,6 +4,7 @@
 import molten
 
 from ... import Pin, config
+from ...constants import EVENT_SAMPLE_RATE_KEY
 from ...ext import AppTypes, http
 from ...propagation.http import HTTPPropagator
 from ...utils.formats import asbool, get_env
@@ -83,6 +84,10 @@ def patch_app_call(wrapped, instance, args, kwargs):
             pin.tracer.context_provider.activate(context)
 
     with pin.tracer.trace('molten.request', service=pin.service, resource=resource) as span:
+        # Configure trace search sample rate
+        if config.molten.event_sample_rate is not None:
+            span.set_tag(EVENT_SAMPLE_RATE_KEY, config.molten.event_sample_rate)
+
         @wrapt.function_wrapper
         def _w_start_response(wrapped, instance, args, kwargs):
             """ Patch respond handling to set metadata """

ddtrace/contrib/pylons/middleware.py

@@ -8,8 +8,11 @@
 from .constants import CONFIG_MIDDLEWARE
 
 from ...compat import reraise
+from ...constants import EVENT_SAMPLE_RATE_KEY
 from ...ext import http, AppTypes
 from ...propagation.http import HTTPPropagator
+from ...settings import config as ddconfig
+
 
 log = logging.getLogger(__name__)
 
@@ -49,6 +52,10 @@ def __call__(self, environ, start_response):
             # set as early as possible when different services share one single agent.
             span.span_type = http.TYPE
 
+            # Configure trace search sample rate
+            if ddconfig.pylons.event_sample_rate is not None:
+                span.set_tag(EVENT_SAMPLE_RATE_KEY, ddconfig.pylons.event_sample_rate)
+
             if not span.sampled:
                 return self.app(environ, start_response)
 

ddtrace/contrib/pyramid/trace.py

@@ -7,8 +7,10 @@
 
 # project
 import ddtrace
+from ...constants import EVENT_SAMPLE_RATE_KEY
 from ...ext import http, AppTypes
 from ...propagation.http import HTTPPropagator
+from ...settings import config
 from .constants import (
     SETTINGS_TRACER,
     SETTINGS_SERVICE,
@@ -76,6 +78,10 @@ def trace_tween(request):
                 if context.trace_id:
                     tracer.context_provider.activate(context)
             with tracer.trace('pyramid.request', service=service, resource='404') as span:
+                # Configure trace search sample rate
+                if config.pyramid.event_sample_rate is not None:
+                    span.set_tag(EVENT_SAMPLE_RATE_KEY, config.pyramid.event_sample_rate)
+
                 setattr(request, DD_SPAN, span)  # used to find the tracer in templates
                 response = None
                 try:

ddtrace/contrib/tornado/handlers.py

@@ -2,8 +2,10 @@
 
 from .constants import CONFIG_KEY, REQUEST_CONTEXT_KEY, REQUEST_SPAN_KEY
 from .stack_context import TracerStackContext
+from ...constants import EVENT_SAMPLE_RATE_KEY
 from ...ext import http
 from ...propagation.http import HTTPPropagator
+from ...settings import config
 
 
 def execute(func, handler, args, kwargs):
@@ -35,6 +37,10 @@ def execute(func, handler, args, kwargs):
             service=service,
             span_type=http.TYPE
         )
+        # Configure trace search sample rate
+        if config.tornado.event_sample_rate is not None:
+            request_span.set_tag(EVENT_SAMPLE_RATE_KEY, config.tornado.event_sample_rate)
+
         setattr(handler.request, REQUEST_SPAN_KEY, request_span)
 
         return func(*args, **kwargs)