Add initial peek into python 3.9 (#274)

cedricziel · Peter Giacomo Lombardo · web-flow · commit dffbb5fc5643 · 2020-12-07T12:44:46.000+01:00
* Add initial peek into python 3.9

This adds a Circle CI job for Python 3.9 on Buster (not Stretch)

CircleCI does not offer a stable 3.9.0 image, yet and RC2
should resemble the final thing until the stable image arrived.

* Update to latest circleci/python:3.9.0 image

* Switch to requirements.txt files for tests

* Install from .txt requirements

* Version limit asynqp

* Use commands in CircleCI config

* Fix syntax errors

* Cleanup 2.7 test runs

* Fix Python version

Co-authored-by: Peter Giacomo Lombardo &lt;nosis@rbx.run&gt;
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -1,8 +1,55 @@
-# Python CircleCI 2.0 configuration file
-#
-# Check https://circleci.com/docs/2.0/language-python/ for more details
-#
-version: 2
+version: 2.1
+
+# More about orbs: https://circleci.com/docs/2.0/using-orbs/
+# orbs:
+#   ruby: circleci/ruby@1.1.2
+
+commands:
+  pip-install-deps-27:
+    parameters:
+      requirements:
+        default: "tests/requirements.txt"
+        type: string
+    steps:
+      - run:
+          name: Install Python 2.7 Dependencies
+          command: |
+            rm -rf venv
+            export PATH=/home/circleci/.local/bin:$PATH
+            pip install --user -U pip setuptools virtualenv wheel
+            virtualenv --python=python2.7 --always-copy venv
+            . venv/bin/activate
+            pip install 'wheel>=0.29.0'
+            pip install -r requirements.txt
+            pip install -r <<parameters.requirements>>
+
+  pip-install-deps:
+    parameters:
+      requirements:
+        default: "tests/requirements.txt"
+        type: string
+    steps:
+      - run:
+          name: Install Python Dependencies
+          command: |
+            python -m venv venv
+            . venv/bin/activate
+            pip install 'wheel>=0.29.0'
+            pip install -r requirements.txt
+            pip install -r <<parameters.requirements>>
+      
+  install-couchbase-deps:
+    steps:
+      - run:
+          name: Install Couchbase Dependencies
+          command: |
+            sudo apt-get update
+            sudo apt install lsb-release -y
+            curl -O https://packages.couchbase.com/releases/couchbase-release/couchbase-release-1.0-6-amd64.deb
+            sudo dpkg -i ./couchbase-release-1.0-6-amd64.deb
+            sudo apt-get update
+            sudo apt install libcouchbase-dev -y
+            
 jobs:
   python27:
     docker:
@@ -15,17 +62,7 @@ jobs:
     working_directory: ~/repo
     steps:
       - checkout
-      - run:
-          name: install dependencies
-          command: |
-            rm -rf venv
-            export PATH=/home/circleci/.local/bin:$PATH
-            pip install --user -U pip setuptools virtualenv
-            virtualenv --python=python2.7 --always-copy venv
-            . venv/bin/activate
-            pip install -U pip
-            python setup.py install_egg_info
-            pip install -e '.[test]'
+      - pip-install-deps-27
       - run:
           name: run tests
           environment:
@@ -36,7 +73,7 @@ jobs:
 
   python38:
     docker:
-      - image: circleci/python:3.7.8-stretch
+      - image: circleci/python:3.8.6
       - image: circleci/postgres:9.6.5-alpine-ram
       - image: circleci/mariadb:10-ram
       - image: circleci/redis:5.0.4
@@ -45,14 +82,27 @@ jobs:
     working_directory: ~/repo
     steps:
       - checkout
+      - pip-install-deps 
       - run:
-          name: install dependencies
+          name: run tests
+          environment:
+            INSTANA_TEST: "true"
           command: |
-            python -m venv venv
             . venv/bin/activate
-            pip install -U pip
-            python setup.py install_egg_info
-            pip install -e '.[test]'
+            pytest -v
+
+  python39:
+    docker:
+      - image: circleci/python:3.9.0-buster
+      - image: circleci/postgres:9.6.5-alpine-ram
+      - image: circleci/mariadb:10-ram
+      - image: circleci/redis:5.0.4
+      - image: rabbitmq:3.5.4
+      - image: circleci/mongo:4.2.3-ram
+    working_directory: ~/repo
+    steps:
+      - checkout
+      - pip-install-deps 
       - run:
           name: run tests
           environment:
@@ -68,20 +118,9 @@ jobs:
     working_directory: ~/repo
     steps:
       - checkout
-      - run:
-          name: install dependencies
-          command: |
-            sudo apt-get update
-            sudo apt install lsb-release -y
-            curl -O https://packages.couchbase.com/releases/couchbase-release/couchbase-release-1.0-6-amd64.deb
-            sudo dpkg -i ./couchbase-release-1.0-6-amd64.deb
-            sudo apt-get update
-            sudo apt install libcouchbase-dev -y
-            python -m venv venv
-            . venv/bin/activate
-            pip install -U pip
-            python setup.py install_egg_info
-            pip install -e '.[test-couchbase]'
+      - install-couchbase-deps
+      - pip-install-deps:
+          requirements: "tests/requirements-couchbase.txt"
       - run:
           name: run tests
           environment:
@@ -98,20 +137,9 @@ jobs:
     working_directory: ~/repo
     steps:
       - checkout
-      - run:
-          name: install dependencies
-          command: |
-            sudo apt-get update
-            sudo apt install lsb-release -y
-            curl -O https://packages.couchbase.com/releases/couchbase-release/couchbase-release-1.0-6-amd64.deb
-            sudo dpkg -i ./couchbase-release-1.0-6-amd64.deb
-            sudo apt-get update
-            sudo apt install libcouchbase-dev -y
-            python -m venv venv
-            . venv/bin/activate
-            pip install -U pip
-            python setup.py install_egg_info
-            pip install -e '.[test-couchbase]'
+      - install-couchbase-deps
+      - pip-install-deps-27:
+          requirements: "tests/requirements-couchbase.txt"
       - run:
           name: run tests
           environment:
@@ -131,17 +159,9 @@ jobs:
     working_directory: ~/repo
     steps:
       - checkout
-      - run:
-          name: install dependencies
-          command: |
-            rm -rf venv
-            export PATH=/home/circleci/.local/bin:$PATH
-            pip install --user -U pip setuptools virtualenv
-            virtualenv --python=python2.7 --always-copy venv
-            . venv/bin/activate
-            pip install -U pip
-            python setup.py install_egg_info
-            pip install -e '.[test-cassandra]'
+      - install-couchbase-deps
+      - pip-install-deps-27:
+          requirements: "tests/requirements-cassandra.txt"
       - run:
           name: run tests
           environment:
@@ -161,14 +181,8 @@ jobs:
     working_directory: ~/repo
     steps:
       - checkout
-      - run:
-          name: install dependencies
-          command: |
-            python -m venv venv
-            . venv/bin/activate
-            pip install -U pip
-            python setup.py install_egg_info
-            pip install -e '.[test-cassandra]'
+      - pip-install-deps:
+          requirements: "tests/requirements-cassandra.txt"
       - run:
           name: run tests
           environment:
@@ -184,14 +198,8 @@ jobs:
     working_directory: ~/repo
     steps:
       - checkout
-      - run:
-          name: install dependencies
-          command: |
-            python -m venv venv
-            . venv/bin/activate
-            pip install -U pip
-            python setup.py install_egg_info
-            pip install -e '.[test-gevent]'
+      - pip-install-deps:
+          requirements: "tests/requirements-gevent.txt"
       - run:
           name: run tests
           environment:
@@ -200,12 +208,14 @@ jobs:
           command: |
             . venv/bin/activate
             pytest -v tests/frameworks/test_gevent.py
+
 workflows:
   version: 2
   build:
     jobs:
       - python27
       - python38
+      - python39
       - py27cassandra
       - py36cassandra
       - gevent38
diff --git a/instana/__init__.py b/instana/__init__.py
@@ -133,9 +133,11 @@ def boot_agent():
             from .instrumentation import asyncio
             from .instrumentation.aiohttp import client
             from .instrumentation.aiohttp import server
-            from .instrumentation import asynqp
             from .instrumentation import boto3_inst
 
+        if sys.version_info >= (3, 5, 3) and sys.version_info < (3, 8, 0):
+            from .instrumentation import asynqp
+
         if sys.version_info[0] < 3:
             from .instrumentation import mysqlpython
             from .instrumentation import webapp2_inst
diff --git a/requirements-test.txt b/requirements-test.txt
diff --git a/setup.py b/setup.py
@@ -71,61 +71,6 @@ def check_setuptools():
                     'django':   ['string = instana:load'],  # deprecated: use same as 'instana'
                     'django19': ['string = instana:load'],  # deprecated: use same as 'instana'
                     },
-      extras_require={
-          'test-gevent': [
-              'flask>=0.12.2',
-              'gevent>=1.4.0',
-              'mock>=2.0.0',
-              'nose>=1.0',
-              'pyramid>=1.2',
-              'pytest>=4.6',
-              'urllib3[secure]!=1.25.0,!=1.25.1,<1.26,>=1.21.1'
-          ],
-          'test-cassandra': [
-              'cassandra-driver==3.20.2',
-              'mock>=2.0.0',
-              'nose>=1.0',
-              'pytest>=4.6',
-              'urllib3[secure]!=1.25.0,!=1.25.1,<1.26,>=1.21.1'
-          ],
-          'test-couchbase': [
-              'couchbase==2.5.9',
-          ],
-          'test': [
-              'aiofiles>=0.5.0;python_version>="3.5"',
-              'aiohttp>=3.5.4;python_version>="3.5"',
-              'asynqp>=0.4;python_version>="3.5"',
-              'boto3>=1.10.0',
-              'celery>=4.1.1',
-              'django>=1.11,<2.2',
-              'fastapi>=0.61.1;python_version>="3.6"',
-              'flask>=0.12.2',
-              'grpcio>=1.18.0',
-              'google-cloud-storage>=1.24.0;python_version>="3.5"',
-              'lxml>=3.4',
-              'mock>=2.0.0',
-              'moto>=1.3.16',
-              'mysqlclient>=1.3.14;python_version>="3.5"',
-              'MySQL-python>=1.2.5;python_version<="2.7"',
-              'nose>=1.0',
-              'PyMySQL[rsa]>=0.9.1',
-              'pyOpenSSL>=16.1.0;python_version<="2.7"',
-              'psycopg2>=2.7.1',
-              'pymongo>=3.7.0',
-              'pyramid>=1.2',
-              'pytest>=4.6',
-              'pytest-celery',
-              'redis>3.0.0',
-              'requests>=2.17.1',
-              'sqlalchemy>=1.1.15',
-              'spyne>=2.9,<=2.12.14',
-              'suds-jurko>=0.6',
-              'tornado>=4.5.3,<6.0',
-              'uvicorn>=0.12.2;python_version>="3.6"',
-              'urllib3[secure]!=1.25.0,!=1.25.1,<1.26,>=1.21.1'
-          ],
-      },
-      test_suite='nose.collector',
       keywords=['performance', 'opentracing', 'metrics', 'monitoring',
                 'tracing', 'distributed-tracing'],
       classifiers=[
diff --git a/tests/clients/test_asynqp.py b/tests/clients/test_asynqp.py
@@ -20,8 +20,10 @@
 else:
     rabbitmq_host = "localhost"
 
-#@pytest.mark.skipif(LooseVersion(sys.version) < LooseVersion('3.5.3'), reason="")
-@pytest.mark.skip("FIXME: Abandoned asynqp is now causing issues in later Python versions.")
+is_unsupported_version = LooseVersion(sys.version) < LooseVersion('3.5.3') \
+    or LooseVersion(sys.version) >= LooseVersion('3.8.0')
+
+@pytest.mark.skipif(is_unsupported_version, reason="Asynqp supports >=3.5.3;<3.8.0")
 class TestAsynqp(unittest.TestCase):
     @asyncio.coroutine
     def connect(self):
diff --git a/tests/conftest.py b/tests/conftest.py
@@ -18,13 +18,15 @@
     collect_ignore_glob.append("*test_gevent*")
 
 if LooseVersion(sys.version) < LooseVersion('3.5.3'):
-    collect_ignore_glob.append("*test_asynqp*")
     collect_ignore_glob.append("*test_aiohttp*")
     collect_ignore_glob.append("*test_async*")
     collect_ignore_glob.append("*test_tornado*")
     collect_ignore_glob.append("*test_grpc*")
     collect_ignore_glob.append("*test_boto3*")
 
+if LooseVersion(sys.version) < LooseVersion('3.5.3') or LooseVersion(sys.version) >= LooseVersion('3.8.0'):
+    collect_ignore_glob.append("*test_asynqp*")
+
 if LooseVersion(sys.version) < LooseVersion('3.6.0'):
     collect_ignore_glob.append("*test_fastapi*")
     collect_ignore_glob.append("*test_starlette*")
diff --git a/tests/requirements-cassandra.txt b/tests/requirements-cassandra.txt
@@ -0,0 +1,5 @@
+cassandra-driver==3.20.2
+mock>=2.0.0
+nose>=1.0
+pytest>=4.6
+urllib3[secure]!=1.25.0,!=1.25.1,<1.26,>=1.21.1
diff --git a/tests/requirements-couchbase.txt b/tests/requirements-couchbase.txt
@@ -0,0 +1 @@
+couchbase==2.5.9
diff --git a/tests/requirements-gevent.txt b/tests/requirements-gevent.txt
@@ -0,0 +1,7 @@
+flask>=0.12.2
+gevent>=1.4.0
+mock>=2.0.0
+nose>=1.0
+pyramid>=1.2
+pytest>=4.6
+urllib3[secure]!=1.25.0,!=1.25.1,<1.26,>=1.21.1
diff --git a/tests/requirements.txt b/tests/requirements.txt
@@ -0,0 +1,31 @@
+aiofiles>=0.5.0;python_version>="3.5"
+aiohttp>=3.5.4;python_version>="3.5"
+asynqp>=0.4;python_version>="3.5"
+boto3>=1.10.0
+celery>=4.1.1
+django>=1.11,<2.2
+fastapi>=0.61.1;python_version>="3.6"
+flask>=0.12.2
+grpcio>=1.18.0
+google-cloud-storage>=1.24.0;python_version>="3.5"
+lxml>=3.4
+mock>=2.0.0
+moto>=1.3.16
+mysqlclient>=1.3.14;python_version>="3.5"
+MySQL-python>=1.2.5;python_version<="2.7"
+nose>=1.0
+PyMySQL[rsa]>=0.9.1
+pyOpenSSL>=16.1.0;python_version<="2.7"
+psycopg2>=2.7.1
+pymongo>=3.7.0
+pyramid>=1.2
+pytest>=4.6
+pytest-celery
+redis>3.0.0
+requests>=2.17.1
+sqlalchemy>=1.1.15
+spyne>=2.9,<=2.12.14
+suds-jurko>=0.6
+tornado>=4.5.3,<6.0
+uvicorn>=0.12.2;python_version>="3.6"
+urllib3[secure]!=1.25.0,!=1.25.1,<1.26,>=1.21.1
