[core/api] Enable buffering on getresponse (#527)

* [core/api] buffer http recv calls

* [core] add regression test for get_connect_response compat function

Author: Kyle-Verhoog

ddtrace/api.py

@@ -6,7 +6,7 @@
 
 # project
 from .encoding import get_encoder, JSONEncoder
-from .compat import httplib, PYTHON_VERSION, PYTHON_INTERPRETER
+from .compat import httplib, PYTHON_VERSION, PYTHON_INTERPRETER, get_connection_response
 
 
 log = logging.getLogger(__name__)
@@ -140,4 +140,4 @@ def _put(self, endpoint, data, count=0):
             headers[TRACE_COUNT_HEADER] = str(count)
 
         conn.request("PUT", endpoint, data, headers)
-        return conn.getresponse()
+        return get_connection_response(conn)

ddtrace/compat.py

@@ -70,6 +70,24 @@ def to_unicode(s):
     return stringify(s)
 
 
+def get_connection_response(conn):
+    """Returns the response for a connection.
+
+    If using Python 2 enable buffering.
+
+    Python 2 does not enable buffering by default resulting in many recv
+    syscalls.
+
+    See:
+    https://bugs.python.org/issue4879
+    https://github.com/python/cpython/commit/3c43fcba8b67ea0cec4a443c755ce5f25990a6cf
+    """
+    if PY2:
+        return conn.getresponse(buffering=True)
+    else:
+        return conn.getresponse()
+
+
 if PY2:
     string_type = basestring
     msgpack_type = basestring

tests/test_compat.py

@@ -3,10 +3,10 @@
 import sys
 
 # Third party
-from nose.tools import eq_, assert_raises
+from nose.tools import eq_, ok_, assert_raises
 
 # Project
-from ddtrace.compat import to_unicode, PY2, reraise
+from ddtrace.compat import to_unicode, PY2, reraise, get_connection_response
 
 
 # Use different test suites for each Python version, this allows us to test the expected
@@ -61,6 +61,16 @@ def test_to_unicode_non_string(self):
             eq_(to_unicode(None), u'None')
             eq_(to_unicode(dict(key='value')), u'{\'key\': \'value\'}')
 
+        def test_get_connection_response(self):
+            """Ensure that buffering is in kwargs."""
+
+            class MockConn(object):
+                def getresponse(self, *args, **kwargs):
+                    ok_('buffering' in kwargs)
+
+            mock = MockConn()
+            get_connection_response(mock)
+
 else:
     class TestCompatPY3(object):
         def test_to_unicode_string(self):
@@ -94,6 +104,16 @@ def test_to_unicode_non_string(self):
             eq_(to_unicode(None), 'None')
             eq_(to_unicode(dict(key='value')), '{\'key\': \'value\'}')
 
+        def test_get_connection_response(self):
+            """Ensure that buffering is NOT in kwargs."""
+
+            class MockConn(object):
+                def getresponse(self, *args, **kwargs):
+                    ok_('buffering' not in kwargs)
+
+            mock = MockConn()
+            get_connection_response(mock)
+
 
 class TestPy2Py3Compat(object):
     """Common tests to ensure functions are both Python 2 and

tests/test_integration.py

@@ -13,7 +13,6 @@
 from ddtrace.ext import http
 from ddtrace.filters import FilterRequestsOnUrl
 from ddtrace.constants import FILTERS_KEY
-from ddtrace.span import Span
 from ddtrace.tracer import Tracer
 from ddtrace.encoding import JSONEncoder, MsgpackEncoder, get_encoder
 from ddtrace.compat import httplib, PYTHON_INTERPRETER, PYTHON_VERSION