Merge pull request #846 from DataDog/0.23-dev

Release 0.23.0

Author: majorgreys

.circleci/config.yml

@@ -62,11 +62,14 @@ jobs:
       # Install required dependencies
       # DEV: `pyopenssl` needed until the following PR is released
       #      https://github.com/pypa/twine/pull/447
-      - run: pip install twine readme_renderer[md] pyopenssl
+      # DEV: `wheel` is needed to run `bdist_wheel`
+      - run: pip install twine readme_renderer[md] pyopenssl wheel
       # Ensure we didn't cache from previous runs
       - run: rm -rf dist/
-      # Ensure package will build
+      # Ensure source package will build
       - run: python setup.py sdist
+      # Ensure wheel will build
+      - run: python setup.py bdist_wheel
       # Ensure package long description is valid and will render
       # https://github.com/pypa/twine/tree/6c4d5ecf2596c72b89b969ccc37b82c160645df8#twine-check
       - run: twine check dist/*
@@ -86,6 +89,20 @@ jobs:
             - tracer.results
       - *save_cache_step
 
+  internal:
+    docker:
+      - *test_runner
+    resource_class: *resource_class
+    steps:
+      - checkout
+      - *restore_cache_step
+      - run: tox -e '{py27,py34,py35,py36}-internal' --result-json /tmp/internal.results
+      - persist_to_workspace:
+          root: /tmp
+          paths:
+            - internal.results
+      - *save_cache_step
+
   opentracer:
     docker:
       - *test_runner
@@ -312,11 +329,13 @@ jobs:
     steps:
       - checkout
       - *restore_cache_step
-      - run: tox -e 'celery_contrib-{py27,py34,py35,py36}-celery{31,40,41,42}-redis{210}' --result-json /tmp/celery.results
+      - run: tox -e 'celery_contrib-{py27,py34,py35,py36}-celery{31}-redis{210}' --result-json /tmp/celery.1.results
+      - run: tox -e 'celery_contrib-{py27,py34,py35,py36}-celery{40,41,42}-{redis210-kombu43,redis320-kombu44}' --result-json /tmp/celery.2.results
       - persist_to_workspace:
           root: /tmp
           paths:
-            - celery.results
+            - celery.1.results
+            - celery.2.results
       - *save_cache_step
 
   elasticsearch:
@@ -1042,6 +1061,9 @@ workflows:
       - integration:
           requires:
             - flake8
+      - internal:
+          requires:
+            - flake8
       - jinja2:
           requires:
             - flake8
@@ -1156,6 +1178,7 @@ workflows:
             - grpc
             - httplib
             - integration
+            - internal
             - jinja2
             - kombu
             - mako

Rakefile

@@ -1,90 +1,3 @@
-desc "Starts all backing services and run all tests"
-task :test do
-  sh "docker-compose up -d | cat"
-  begin
-    sh "tox"
-  ensure
-    sh "docker-compose kill"
-  end
-  sh "python -m tests.benchmark"
-end
-
-desc 'CI dependent task; tests in parallel'
-task:test_parallel do
-
-  begin
-    test_cassandra = sh "git diff-tree --no-commit-id --name-only -r HEAD | grep ddtrace/contrib/cassandra"
-  rescue StandardError => e
-    test_cassandra = false
-  end
-
-  sh "docker-compose up -d | cat"
-
-  # If cassandra hasn't been changed ignore cassandra tests
-  if not test_cassandra
-    n_total_envs = `tox -l | grep -v cassandra | wc -l`.to_i
-    envs = 'tox -l | grep -v cassandra | tr \'\n\' \',\''
-  else
-    n_total_envs = `tox -l | wc -l`.to_i
-    envs = 'tox -l | tr \'\n\' \',\''
-  end
-
-  circle_node_tot = ENV['CIRCLE_NODE_TOTAL'].to_i
-  n_envs_chunk = n_total_envs / circle_node_tot
-  env_list_start = 1
-  env_list_end = n_envs_chunk
-  begin
-    for node_index in 0..circle_node_tot
-      if ENV['CIRCLE_NODE_INDEX'].to_i == node_index then
-        # Node 0 already does as second task wait test, the others will require it to ensure db connections
-        if node_index >= 1 then
-          sh "tox -e wait"
-        end
-        sh "#{envs} | cut -d, -f#{env_list_start}-#{env_list_end} | xargs tox -e"
-      end
-      env_list_start = env_list_end + 1
-      env_list_end = env_list_end + n_envs_chunk
-    end
-  ensure
-    sh "docker-compose kill"
-  end
-
-  sh "python -m tests.benchmark"
-end
-
-desc "Run tests with envs matching the given pattern."
-task :"test:envs", [:grep] do |t, args|
-  pattern = args[:grep]
-  if !pattern
-    puts 'specify a pattern like rake test:envs["py27.*mongo"]'
-  else
-    sh "tox -l | grep '#{pattern}' | xargs tox -e"
-  end
-end
-
-namespace :docker do
-  task :up do
-    sh "docker-compose up -d | cat"
-  end
-
-  task :down do
-    sh "docker-compose down"
-  end
-end
-
-
-desc "install the library in dev mode"
-task :dev do
-  sh "pip uninstall -y ddtrace"
-  sh "pip install -e ."
-end
-
-desc "remove artifacts"
-task :clean do
-  sh 'python setup.py clean'
-  sh 'rm -rf build *egg* *.whl dist'
-end
-
 desc "build the docs"
 task :docs do
     sh "pip install sphinx"
@@ -94,7 +7,6 @@ task :docs do
 end
 
 # Deploy tasks
-S3_BUCKET = 'pypi.datadoghq.com'
 S3_DIR = ENV['S3_DIR']
 
 desc "release the a new wheel"
@@ -105,7 +17,8 @@ task :'release:wheel' do
   #  - aws s3 cp dist/*.whl s3://pypi.datadoghq.com/#{s3_dir}/
   fail "Missing environment variable S3_DIR" if !S3_DIR or S3_DIR.empty?
 
-  sh "mkwheelhouse s3://#{S3_BUCKET}/#{S3_DIR}/ ."
+  # Use custom mkwheelhouse script to build and upload an sdist to S3 bucket
+  sh "scripts/mkwheelhouse"
 end
 
 desc "release the docs website"
@@ -169,60 +82,3 @@ namespace :pypi do
     sh "twine upload #{build}"
   end
 end
-
-namespace :version do
-
-  def get_version()
-    return `python setup.py --version`.strip()
-  end
-
-  def set_version(old, new)
-    branch = `git name-rev --name-only HEAD`.strip()
-    if  branch != "master"
-      puts "you should only tag the master branch"
-      return
-    end
-    msg = "bumping version #{old} => #{new}"
-    path = "ddtrace/__init__.py"
-    sh "sed -i 's/#{old}/#{new}/' #{path}"
-    sh "git commit -m '#{msg}' #{path}"
-    sh "git tag v#{new}"
-    puts "Verify everything looks good, then `git push && git push --tags`"
-  end
-
-  def inc_version_num(version, type)
-    split = version.split(".").map{|v| v.to_i}
-    if type == 'bugfix'
-      split[2] += 1
-    elsif type == 'minor'
-       split[1] += 1
-       split[2] = 0
-    elsif type == 'major'
-       split[0] += 1
-       split[1] = 0
-       split[2] = 0
-    end
-    return split.join(".")
-  end
-
-  def inc_version(type)
-    old = get_version()
-    new = inc_version_num(old, type)
-    set_version(old, new)
-  end
-
-  desc "Cut a new bugfix release"
-  task :bugfix do
-    inc_version("bugfix")
-  end
-
-  desc "Cut a new minor release"
-  task :minor do
-    inc_version("minor")
-  end
-
-  task :major do
-    inc_version("major")
-  end
-
-end

ddtrace/__init__.py

@@ -4,7 +4,7 @@
 from .tracer import Tracer
 from .settings import config
 
-__version__ = '0.22.0'
+__version__ = '0.23.0'
 
 # a global tracer instance with integration settings
 tracer = Tracer()

ddtrace/api.py

@@ -1,16 +1,17 @@
 # stdlib
-import logging
 import time
 import ddtrace
 from json import loads
 
 # project
 from .encoding import get_encoder, JSONEncoder
 from .compat import httplib, PYTHON_VERSION, PYTHON_INTERPRETER, get_connection_response
+from .internal.logger import get_logger
+from .payload import Payload
 from .utils.deprecation import deprecated
 
 
-log = logging.getLogger(__name__)
+log = get_logger(__name__)
 
 TRACE_COUNT_HEADER = 'X-Datadog-Trace-Count'
 
@@ -147,9 +148,13 @@ def _downgrade(self):
     def send_traces(self, traces):
         if not traces:
             return
+
         start = time.time()
-        data = self._encoder.encode_traces(traces)
-        response = self._put(self._traces, data, len(traces))
+        payload = Payload(encoder=self._encoder)
+        for trace in traces:
+            payload.add_trace(trace)
+
+        response = self._put(self._traces, payload.get_payload(), payload.length)
 
         # the API endpoint is not available so we should downgrade the connection and re-try the call
         if response.status in [404, 415] and self._fallback:

ddtrace/bootstrap/sitecustomize.py

@@ -9,6 +9,7 @@
 import logging
 
 from ddtrace.utils.formats import asbool, get_env
+from ddtrace.internal.logger import get_logger
 
 logs_injection = asbool(get_env('logs', 'injection'))
 DD_LOG_FORMAT = '%(asctime)s %(levelname)s [%(name)s] [%(filename)s:%(lineno)d] {}- %(message)s'.format(
@@ -32,7 +33,7 @@
 else:
     logging.basicConfig(format=DD_LOG_FORMAT)
 
-log = logging.getLogger(__name__)
+log = get_logger(__name__)
 
 EXTRA_PATCHED_MODULES = {
     "bottle": True,

ddtrace/commands/ddtrace_run.py

@@ -10,6 +10,11 @@
 if debug and debug.lower() == "true":
     logging.basicConfig(level=logging.DEBUG)
 
+# Do not use `ddtrace.internal.logger.get_logger` here
+# DEV: It isn't really necessary to use `DDLogger` here so we want to
+#        defer importing `ddtrace` until we actually need it.
+#      As well, no actual rate limiting would apply here since we only
+#        have a few logged lines
 log = logging.getLogger(__name__)
 
 USAGE = """

ddtrace/constants.py

@@ -1,7 +1,7 @@
 FILTERS_KEY = 'FILTERS'
 SAMPLE_RATE_METRIC_KEY = '_sample_rate'
 SAMPLING_PRIORITY_KEY = '_sampling_priority_v1'
+ANALYTICS_SAMPLE_RATE_KEY = '_dd1.sr.eausr'
 ORIGIN_KEY = '_dd.origin'
-EVENT_SAMPLE_RATE_KEY = '_dd1.sr.eausr'
 
-NUMERIC_TAGS = (EVENT_SAMPLE_RATE_KEY, )
+NUMERIC_TAGS = (ANALYTICS_SAMPLE_RATE_KEY, )

ddtrace/context.py

@@ -1,10 +1,10 @@
-import logging
 import threading
 
 from .constants import SAMPLING_PRIORITY_KEY, ORIGIN_KEY
+from .internal.logger import get_logger
 from .utils.formats import asbool, get_env
 
-log = logging.getLogger(__name__)
+log = get_logger(__name__)
 
 
 class Context(object):

ddtrace/contrib/aiobotocore/__init__.py

@@ -1,7 +1,7 @@
 """
 The aiobotocore integration will trace all AWS calls made with the ``aiobotocore``
 library. This integration isn't enabled when applying the default patching.
-To enable it, you must run ``patch_all(botocore=True)``
+To enable it, you must run ``patch_all(aiobotocore=True)``
 
 ::
 

ddtrace/contrib/aiobotocore/patch.py

@@ -1,9 +1,11 @@
 import asyncio
 from ddtrace.vendor import wrapt
+from ddtrace import config
 import aiobotocore.client
 
 from aiobotocore.endpoint import ClientResponseContentProxy
 
+from ...constants import ANALYTICS_SAMPLE_RATE_KEY
 from ...pin import Pin
 from ...ext import http, aws
 from ...compat import PYTHON_VERSION_INFO
@@ -117,4 +119,10 @@ def _wrapped_api_call(original_func, instance, args, kwargs):
         if request_id2:
             span.set_tag('aws.requestid2', request_id2)
 
+        # set analytics sample rate
+        span.set_tag(
+            ANALYTICS_SAMPLE_RATE_KEY,
+            config.aiobotocore.get_analytics_sample_rate()
+        )
+
         return result