[tests] Add benchmarks (#952)

* Move to pytest-benchmarks and add to CI

* Add tests for more tracer functionality

* Save benchmark results to CircleCI artifact

* Remove outdated documentation

Author: majorgreys

.circleci/config.yml

@@ -675,6 +675,24 @@ jobs:
       - *persist_to_workspace_step
       - *save_cache_step
 
+  benchmarks:
+    docker:
+      - *test_runner
+    resource_class: *resource_class
+    steps:
+      - checkout
+      - *restore_cache_step
+      - run:
+          command: |
+            mkdir -p /tmp/test-reports
+            tox -e 'benchmarks-{py27,py34,py35,py36,py37}' --result-json /tmp/benchmarks.results -- --benchmark-storage=file:///tmp/test-reports/ --benchmark-autosave
+      - store_test_results:
+          path: /tmp/test-reports
+      - store_artifacts:
+          path: /tmp/test-reports
+      - *persist_to_workspace_step
+      - *save_cache_step
+
   deploy_dev:
     # build the master branch releasing development docs and wheels
     docker:
@@ -806,6 +824,9 @@ workflows:
       - algoliasearch:
           requires:
             - flake8
+      - benchmarks:
+          requires:
+            - flake8
       - boto:
           requires:
             - flake8
@@ -951,6 +972,7 @@ workflows:
             - aiopg
             - asyncio
             - algoliasearch
+            - benchmarks
             - boto
             - bottle
             - cassandra

README.md

@@ -88,12 +88,3 @@ the CLI can be found at https://circleci.com/docs/2.0/local-cli/.
 After installing the `circleci` CLI, you can run jobs by name. For example:
 
     $ circleci build --job django
-
-
-### Benchmarking
-
-When two or more approaches must be compared, please write a benchmark in the
-[benchmark.py](tests/benchmark.py) module so that we can measure the efficiency
-of the algorithm. To run your benchmark, just:
-
-    $ python -m tests.benchmark

tests/benchmark.py

@@ -1,86 +1,92 @@
-import timeit
-
 from ddtrace import Tracer
+import pytest
 
 from .test_tracer import DummyWriter
-from os import getpid
-
 
-REPEAT = 10
-NUMBER = 10000
 
+@pytest.fixture
+def tracer():
+    tracer = Tracer()
+    tracer.writer = DummyWriter()
+    return tracer
 
-def trace_error(tracer):
-    # explicit vars
-    with tracer.trace('a', service='s', resource='r', span_type='t'):
-        1 / 0
 
+def test_tracer_context(benchmark, tracer):
+    def func(tracer):
+        with tracer.trace('a', service='s', resource='r', span_type='t'):
+            pass
 
-def benchmark_tracer_trace():
-    tracer = Tracer()
-    tracer.writer = DummyWriter()
+    benchmark(func, tracer)
 
-    # testcase
-    def trace(tracer):
-        # explicit vars
-        with tracer.trace('a', service='s', resource='r', span_type='t') as s:
-            s.set_tag('a', 'b')
-            s.set_tag('b', 1)
-            with tracer.trace('another.thing'):
-                pass
-            with tracer.trace('another.thing'):
-                pass
-
-    # benchmark
-    print('## tracer.trace() benchmark: {} loops ##'.format(NUMBER))
-    timer = timeit.Timer(lambda: trace(tracer))
-    result = timer.repeat(repeat=REPEAT, number=NUMBER)
-    print('- trace execution time: {:8.6f}'.format(min(result)))
-
-
-def benchmark_tracer_wrap():
-    tracer = Tracer()
-    tracer.writer = DummyWriter()
 
-    # testcase
+def test_tracer_wrap_staticmethod(benchmark, tracer):
     class Foo(object):
         @staticmethod
         @tracer.wrap()
-        def s():
+        def func():
             return 0
 
+    f = Foo()
+    benchmark(f.func)
+
+
+def test_tracer_wrap_classmethod(benchmark, tracer):
+    class Foo(object):
         @classmethod
         @tracer.wrap()
-        def c(cls):
+        def func(cls):
             return 0
 
+    f = Foo()
+    benchmark(f.func)
+
+
+def test_tracer_wrap_instancemethod(benchmark, tracer):
+    class Foo(object):
         @tracer.wrap()
-        def m(self):
+        def func(self):
             return 0
 
     f = Foo()
+    benchmark(f.func)
+
+
+def test_tracer_start_span(benchmark, tracer):
+    benchmark(tracer.start_span, 'benchmark')
+
+
+def test_tracer_start_finish_span(benchmark, tracer):
+    def func(tracer):
+        s = tracer.start_span('benchmark')
+        s.finish()
+
+    benchmark(func, tracer)
+
+
+def test_trace_simple_trace(benchmark, tracer):
+    def func(tracer):
+        with tracer.trace('parent'):
+            for i in range(5):
+                with tracer.trace('child') as c:
+                    c.set_tag('i', i)
+
+    benchmark(func, tracer)
+
+
+def test_tracer_large_trace(benchmark, tracer):
+    import random
+
+    # generate trace with 1024 spans
+    @tracer.wrap()
+    def func(tracer, level=0):
+        span = tracer.current_span()
+
+        # do some work
+        num = random.randint(1, 10)
+        span.set_tag('num', num)
+
+        if level < 10:
+            func(tracer, level+1)
+            func(tracer, level+1)
 
-    # benchmark
-    print('## tracer.trace() wrapper benchmark: {} loops ##'.format(NUMBER))
-    timer = timeit.Timer(f.s)
-    result = timer.repeat(repeat=REPEAT, number=NUMBER)
-    print('- staticmethod execution time: {:8.6f}'.format(min(result)))
-    timer = timeit.Timer(f.c)
-    result = timer.repeat(repeat=REPEAT, number=NUMBER)
-    print('- classmethod execution time: {:8.6f}'.format(min(result)))
-    timer = timeit.Timer(f.m)
-    result = timer.repeat(repeat=REPEAT, number=NUMBER)
-    print('- method execution time: {:8.6f}'.format(min(result)))
-
-
-def benchmark_getpid():
-    timer = timeit.Timer(getpid)
-    result = timer.repeat(repeat=REPEAT, number=NUMBER)
-    print('## getpid wrapper benchmark: {} loops ##'.format(NUMBER))
-    print('- getpid execution time: {:8.6f}'.format(min(result)))
-
-
-if __name__ == '__main__':
-    benchmark_tracer_wrap()
-    benchmark_tracer_trace()
-    benchmark_getpid()
+    benchmark(func, tracer)

tox.ini

@@ -118,6 +118,7 @@ envlist =
     py37-opentracer_gevent-gevent{13,14}
 # Unit tests: pytest based test suite that do not require any additional dependency
     unit_tests-{py27,py34,py35,py36,py37}
+    benchmarks-{py27,py34,py35,py36,py37}
 
 [testenv]
 usedevelop = True
@@ -134,6 +135,7 @@ deps =
     !ddtracerun: wrapt
     !msgpack03-!msgpack04-!msgpack05-!ddtracerun: msgpack-python
     pytest>=3
+    pytest-benchmark
     opentracing
     psutil
 # test dependencies installed in all envs
@@ -389,6 +391,7 @@ commands =
     test_logging: pytest {posargs} tests/contrib/logging/
 # Unit tests: pytest based test suite that do not require any additional dependency.
     unit_tests: pytest {posargs} tests/unit
+    benchmarks: pytest --benchmark-only {posargs} tests/benchmark.py
 
 setenv =
     DJANGO_SETTINGS_MODULE = app.settings