New gevent Instrumentation (#232)

* Code comment update

* Function docs and add get_spans_by_filter helper

* Update tests to follow helper name change

* Initial gevent instrumentation

* Use cloned scopes to propagate context

* Updated tests with dedicated gevent run

* Update test python versions and workflow jobs

* CircleCI config cleanup; Break tests out

* Fix Cassandra tests dependencies

* Conditional gevent imports

* gevent tests require flask

* Use stretch image for 3.8 tests

* Add version check to gevent intrumentation

* gevent based agent booting

* Linter improvements and refactoring

* Update warning re: uWSGI threads.  Now gevent support.

* Add tests for imap unordered

* Code comments and updated debug log messages

Author: pglombardo

.circleci/config.yml

@@ -7,29 +7,15 @@ jobs:
   python27:
     docker:
       - image: circleci/python:2.7.15
-
-      # Specify service dependencies here if necessary
-      # CircleCI maintains a library of pre-built images
-      # documented at https://circleci.com/docs/2.0/circleci-images/
       - image: circleci/postgres:9.6.5-alpine-ram
       - image: circleci/mariadb:10.1-ram
       - image: circleci/redis:5.0.4
       - image: rabbitmq:3.5.4
       - image: couchbase/server-sandbox:5.5.0
       - image: circleci/mongo:4.2.3-ram
-
     working_directory: ~/repo
-
     steps:
       - checkout
-
-      # Download and cache dependencies
-      - restore_cache:
-          keys:
-            - v1-dependencies-{{ checksum "requirements.txt" }}
-            # fallback to using the latest cache if no exact match is found
-            - v1-dependencies-
-
       - run:
           name: install dependencies
           command: |
@@ -46,45 +32,25 @@ jobs:
             . venv/bin/activate
             pip install -U pip
             python setup.py install_egg_info
-            pip install -r requirements-test.txt
-
-      - save_cache:
-          paths:
-            - ./venv
-          key: v1-dependencies-{{ checksum "requirements.txt" }}
-
+            pip install -e '.[test]'
       - run:
           name: run tests
           command: |
             . venv/bin/activate
             python runtests.py
 
-  python35:
+  python38:
     docker:
-      - image: circleci/python:3.5.6
-
-      # Specify service dependencies here if necessary
-      # CircleCI maintains a library of pre-built images
-      # documented at https://circleci.com/docs/2.0/circleci-images/
+      - image: circleci/python:3.7.7-stretch
       - image: circleci/postgres:9.6.5-alpine-ram
       - image: circleci/mariadb:10-ram
       - image: circleci/redis:5.0.4
       - image: rabbitmq:3.5.4
       - image: couchbase/server-sandbox:5.5.0
       - image: circleci/mongo:4.2.3-ram
-
     working_directory: ~/repo
-
     steps:
       - checkout
-
-      # Download and cache dependencies
-      - restore_cache:
-          keys:
-            - v1-dependencies-{{ checksum "requirements.txt" }}
-            # fallback to using the latest cache if no exact match is found
-            - v1-dependencies-
-
       - run:
           name: install dependencies
           command: |
@@ -98,175 +64,89 @@ jobs:
             . venv/bin/activate
             pip install -U pip
             python setup.py install_egg_info
-            pip install -r requirements.txt
-            pip install -r requirements-test.txt
-
-      - save_cache:
-          paths:
-            - ./venv
-          key: v1-dependencies-{{ checksum "requirements.txt" }}
-
+            pip install -e '.[test]'
       - run:
           name: run tests
           command: |
             . venv/bin/activate
             python runtests.py
 
-  python36:
+  py27cassandra:
     docker:
-      - image: circleci/python:3.6.8
-
-      # Specify service dependencies here if necessary
-      # CircleCI maintains a library of pre-built images
-      # documented at https://circleci.com/docs/2.0/circleci-images/
-      - image: circleci/postgres:9.6.5-alpine-ram
-      - image: circleci/mariadb:10-ram
-      - image: circleci/redis:5.0.4
-      - image: rabbitmq:3.5.4
-      - image: couchbase/server-sandbox:5.5.0
-      - image: circleci/mongo:4.2.3-ram
-
+      - image: circleci/python:2.7.15
+      - image: circleci/cassandra:3.10
+        environment:
+          MAX_HEAP_SIZE: 2048m
+          HEAP_NEWSIZE: 512m
     working_directory: ~/repo
-
     steps:
       - checkout
-
-      # Download and cache dependencies
-      - restore_cache:
-          keys:
-            - v1-dependencies-{{ checksum "requirements.txt" }}
-            # fallback to using the latest cache if no exact match is found
-            - v1-dependencies-
-
       - run:
           name: install dependencies
           command: |
-            sudo apt-get update
-            sudo apt install lsb-release -y
-            curl -O https://packages.couchbase.com/releases/couchbase-release/couchbase-release-1.0-6-amd64.deb
-            sudo dpkg -i ./couchbase-release-1.0-6-amd64.deb
-            sudo apt-get update
-            sudo apt install libcouchbase-dev -y
-            python -m venv venv
+            rm -rf venv
+            export PATH=/home/circleci/.local/bin:$PATH
+            pip install --user -U pip setuptools virtualenv
+            virtualenv --python=python2.7 --always-copy venv
             . venv/bin/activate
             pip install -U pip
             python setup.py install_egg_info
-            pip install -r requirements.txt
-            pip install -r requirements-test.txt
-
-      - save_cache:
-          paths:
-            - ./venv
-          key: v1-dependencies-{{ checksum "requirements.txt" }}
-
+            pip install -e '.[test-cassandra]'
       - run:
           name: run tests
           command: |
             . venv/bin/activate
-            python runtests.py
+            CASSANDRA_TEST=1 nosetests -v tests/test_cassandra-driver.py:TestCassandra
 
-  py27cassandra:
+  py36cassandra:
     docker:
-      - image: circleci/python:2.7.15
+      - image: circleci/python:3.6.8
       - image: circleci/cassandra:3.10
         environment:
           MAX_HEAP_SIZE: 2048m
           HEAP_NEWSIZE: 512m
-
     working_directory: ~/repo
-
     steps:
       - checkout
-
-      # Download and cache dependencies
-      - restore_cache:
-          keys:
-            - v1-dependencies-{{ checksum "requirements.txt" }}
-            # fallback to using the latest cache if no exact match is found
-            - v1-dependencies-
-
       - run:
           name: install dependencies
           command: |
-            sudo apt-get update
-            sudo apt install lsb-release -y
-            curl -O https://packages.couchbase.com/releases/couchbase-release/couchbase-release-1.0-6-amd64.deb
-            sudo dpkg -i ./couchbase-release-1.0-6-amd64.deb
-            sudo apt-get update
-            sudo apt install libcouchbase-dev -y
-            rm -rf venv
-            export PATH=/home/circleci/.local/bin:$PATH
-            pip install --user -U pip setuptools virtualenv
-            virtualenv --python=python2.7 --always-copy venv
+            python -m venv venv
             . venv/bin/activate
             pip install -U pip
             python setup.py install_egg_info
-            pip install -r requirements-test.txt
-
-      - save_cache:
-          paths:
-            - ./venv
-          key: v1-dependencies-{{ checksum "requirements.txt" }}
-
+            pip install -e '.[test-cassandra]'
       - run:
           name: run tests
           command: |
             . venv/bin/activate
-            nosetests -v tests/test_cassandra-driver.py:TestCassandra
+            CASSANDRA_TEST=1 nosetests -v tests/test_cassandra-driver.py:TestCassandra
 
-  py36cassandra:
+  gevent38:
     docker:
-      - image: circleci/python:3.6.8
-      - image: circleci/cassandra:3.10
-        environment:
-          MAX_HEAP_SIZE: 2048m
-          HEAP_NEWSIZE: 512m
-
+      - image: circleci/python:3.8.2
     working_directory: ~/repo
-
     steps:
       - checkout
-
-      # Download and cache dependencies
-      - restore_cache:
-          keys:
-            - v1-dependencies-{{ checksum "requirements.txt" }}
-            # fallback to using the latest cache if no exact match is found
-            - v1-dependencies-
-
       - run:
           name: install dependencies
           command: |
-            sudo apt-get update
-            sudo apt install lsb-release -y
-            curl -O https://packages.couchbase.com/releases/couchbase-release/couchbase-release-1.0-6-amd64.deb
-            sudo dpkg -i ./couchbase-release-1.0-6-amd64.deb
-            sudo apt-get update
-            sudo apt install libcouchbase-dev -y
             python -m venv venv
             . venv/bin/activate
             pip install -U pip
             python setup.py install_egg_info
-            pip install -r requirements.txt
-            pip install -r requirements-test.txt
-
-      - save_cache:
-          paths:
-            - ./venv
-          key: v1-dependencies-{{ checksum "requirements.txt" }}
-
+            pip install -e '.[test-gevent]'
       - run:
           name: run tests
           command: |
             . venv/bin/activate
-            nosetests -v tests/test_cassandra-driver.py:TestCassandra
-
+            GEVENT_TEST=1 nosetests -v tests/test_gevent.py
 workflows:
   version: 2
   build:
     jobs:
       - python27
-      - python35
-      - python36
+      - python38
       - py27cassandra
       - py36cassandra
+      - gevent38

instana/__init__.py

@@ -95,6 +95,16 @@ def lambda_handler(event, context):
             print("Couldn't determine and locate default function handler: %s.%s", module_name, function_name)
 
 
+def boot_agent_later():
+    """ Executes <boot_agent> in the future! """
+    if 'gevent' in sys.modules:
+        import gevent
+        gevent.spawn_later(2.0, boot_agent)
+    else:
+        t = Timer(2.0, boot_agent)
+        t.start()
+
+
 def boot_agent():
     """Initialize the Instana agent and conditionally load auto-instrumentation."""
     # Disable all the unused-import violations in this function
@@ -122,6 +132,7 @@ def boot_agent():
         from .instrumentation import cassandra_inst
         from .instrumentation import couchbase_inst
         from .instrumentation import flask
+        from .instrumentation import gevent_inst
         from .instrumentation import grpcio
         from .instrumentation.tornado import client
         from .instrumentation.tornado import server
@@ -173,7 +184,6 @@ def boot_agent():
 else:
     if "INSTANA_MAGIC" in os.environ:
         # If we're being loaded into an already running process, then delay agent initialization
-        t = Timer(2.0, boot_agent)
-        t.start()
+        boot_agent_later()
     else:
         boot_agent()

instana/hooks/hook_uwsgi.py

@@ -15,9 +15,9 @@
     opt_master = uwsgi.opt.get('master', False)
     opt_lazy_apps = uwsgi.opt.get('lazy-apps', False)
 
-    if uwsgi.opt.get('enable-threads', False) is False:
-        logger.warn("Required: uWSGI threads are not enabled.  " +
-                    "Please enable by using the uWSGI --enable-threads option.")
+    if uwsgi.opt.get('enable-threads', False) is False and uwsgi.opt.get('gevent', False) is False:
+        logger.warn("Required: Neither uWSGI threads or gevent is enabled. " +
+                    "Please enable by using the uWSGI --enable-threads or --gevent option.")
 
     if opt_master and opt_lazy_apps is False:
         # --master is supplied in uWSGI options (otherwise uwsgidecorators package won't be available)

instana/instrumentation/gevent_inst.py

@@ -0,0 +1,44 @@
+"""
+Instrumentation for the gevent package.
+"""
+from __future__ import absolute_import
+
+import sys
+from ..log import logger
+from ..singletons import tracer
+
+
+def instrument_gevent():
+    """ Adds context propagation to gevent greenlet spawning """
+    try:
+        logger.debug("Instrumenting gevent")
+
+        import gevent
+        from opentracing.scope_managers.gevent import GeventScopeManager
+        from opentracing.scope_managers.gevent import _GeventScope
+
+        def spawn_callback(new_greenlet):
+            """ Handles context propagation for newly spawning greenlets """
+            parent_scope = tracer.scope_manager.active
+            if parent_scope is not None:
+                # New greenlet, new clean slate.  Clone and make active in this new greenlet
+                # the currently active scope (but don't finish() the span on close - it's a
+                # clone/not the original and we don't want to close it prematurely)
+                # TODO: Change to our own ScopeManagers
+                parent_scope_clone = _GeventScope(parent_scope.manager, parent_scope.span, finish_on_close=False)
+                tracer._scope_manager._set_greenlet_scope(parent_scope_clone, new_greenlet)
+
+        logger.debug(" -> Updating tracer to use gevent based context management")
+        tracer._scope_manager = GeventScopeManager()
+        gevent.Greenlet.add_spawn_callback(spawn_callback)
+    except:
+        logger.debug("instrument_gevent: ", exc_info=True)
+
+
+if 'gevent' in sys.modules:
+    if sys.modules['gevent'].version_info < (1, 4):
+        logger.debug("gevent < 1.4 detected.  The Instana package supports gevent versions 1.4 and greater.")
+    else:
+        instrument_gevent()
+else:
+    logger.debug("Instrumenting gevent: gevent not detected or loaded.  Nothing done.")

instana/instrumentation/urllib3.py

@@ -13,8 +13,8 @@
 
     def collect(instance, args, kwargs):
         """ Build and return a fully qualified URL for this request """
+        kvs = dict()
         try:
-            kvs = dict()
             kvs['host'] = instance.host
             kvs['port'] = instance.port
 
@@ -58,7 +58,6 @@ def collect_response(scope, response):
         except Exception:
             logger.debug("collect_response", exc_info=True)
 
-
     @wrapt.patch_function_wrapper('urllib3', 'HTTPConnectionPool.urlopen')
     def urlopen_with_instana(wrapped, instance, args, kwargs):
         parent_span = tracer.active_span

instana/meter.py

@@ -133,7 +133,7 @@ def start(self):
         """
         This function can be called at first boot or after a fork.  In either case, it will
         assure that the Meter is in a proper state (via reset()) and spawn a new background
-        thread to periodically report queued spans
+        thread to periodically report the metrics payload.
 
         Note that this will abandon any previous thread object that (in the case of an `os.fork()`)
         should no longer exist in the forked process.