Add Python 3.11 Support (#654)

* Add py311 tests

* Fix typo

* Added 3.11 support for aiohttp framework

Co-authored-by: Timothy Pansino <[email protected]>

* Set up environment to run Python 3.11

Co-authored-by: Timothy Pansino <[email protected]>

* Add Python 3.11 support for agent_features

Co-authored-by: Timothy Pansino <[email protected]>

* Partial Python 3.11 support added for Tornado

Co-authored-by: Timothy Pansino <[email protected]>

* Adjust postgres versions

* Fix tornado install path locally

* Remove aioredis py311 tests

* Update 3.11 to dev in tests

* Fix sanic instrumentation and imp/importlib deprecation

Co-authored-by: Timothy Pansino <[email protected]>

* Simplify wheel build options

* Update cibuildwheel for 3.11

* Remove falconmaster py311 test

Co-authored-by: Lalleh Rafeei <[email protected]>
Co-authored-by: Timothy Pansino <[email protected]>

Author: TimPansino

.devcontainer/dotfiles

@@ -0,0 +1 @@
+Subproject commit 4d575e4d60a9f195f9f315dde7f380a5ae26e27d

.github/actions/setup-python-matrix/action.yml

@@ -33,6 +33,11 @@ runs:
         python-version: "3.10"
         architecture: x64
 
+    - uses: actions/setup-python@v3
+      with:
+        python-version: "3.11-dev"
+        architecture: x64
+
     - uses: actions/setup-python@v3
       with:
         python-version: "2.7"

.github/workflows/deploy-python.yml

@@ -54,10 +54,10 @@ jobs:
           CIBW_ENVIRONMENT: "LD_LIBRARY_PATH=/opt/rh/=vtoolset-8/root/usr/lib64:/opt/rh/devtoolset-8/root/usr/lib:/opt/rh/devtoolset-8/root/usr/lib64/dyninst:/opt/rh/devtoolset-8/root/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib"
 
       - name: Build Manylinux Wheels (Python 3)
-        uses: pypa/cibuildwheel@v2.1.3
+        uses: pypa/cibuildwheel@v2.11.1
         env:
           CIBW_PLATFORM: linux
-          CIBW_BUILD: cp37-manylinux_aarch64 cp38-manylinux_aarch64 cp39-manylinux_aarch64 cp310-manylinux_aarch64 cp37-manylinux_x86_64 cp38-manylinux_x86_64 cp39-manylinux_x86_64 cp310-manylinux_x86_64
+          CIBW_BUILD: cp37-manylinux* cp38-manylinux* cp39-manylinux* cp310-manylinux* cp311-manylinux*
           CIBW_ARCHS: x86_64 aarch64
           CIBW_ENVIRONMENT: "LD_LIBRARY_PATH=/opt/rh/devtoolset-8/root/usr/lib64:/opt/rh/devtoolset-8/root/usr/lib:/opt/rh/devtoolset-8/root/usr/lib64/dyninst:/opt/rh/devtoolset-8/root/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib"
 

newrelic/hooks/framework_aiohttp.py

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import asyncio
+
 import inspect
 import itertools
 
@@ -163,9 +163,8 @@ def _bind_params(request):
 
 @function_wrapper
 def _nr_aiohttp_wrap_middleware_(wrapped, instance, args, kwargs):
-    @asyncio.coroutine
-    def _inner():
-        result = yield from wrapped(*args, **kwargs)
+    async def _inner():
+        result = await wrapped(*args, **kwargs)
         return function_trace()(result)
 
     return _inner()
@@ -221,10 +220,9 @@ def _nr_aiohttp_add_cat_headers_(wrapped, instance, args, kwargs):
 
     if is_coroutine_callable(wrapped):
 
-        @asyncio.coroutine
-        def new_coro():
+        async def new_coro():
             try:
-                result = yield from wrapped(*args, **kwargs)
+                result = await wrapped(*args, **kwargs)
                 return result
             finally:
                 instance.headers = tmp
@@ -267,10 +265,9 @@ def _nr_aiohttp_request_wrapper_(wrapped, instance, args, kwargs):
     method, url = _bind_request(*args, **kwargs)
     trace = ExternalTrace("aiohttp", str(url), method)
 
-    @asyncio.coroutine
-    def _coro():
+    async def _coro():
         try:
-            response = yield from wrapped(*args, **kwargs)
+            response = await wrapped(*args, **kwargs)
 
             try:
                 trace.process_response_headers(response.headers.items())
@@ -332,14 +329,10 @@ def _nr_request_wrapper(wrapped, instance, args, kwargs):
 
     coro = wrapped(*args, **kwargs)
 
-    if hasattr(coro, "__await__"):
-        coro = coro.__await__()
-
-    @asyncio.coroutine
-    def _coro(*_args, **_kwargs):
+    async def _coro(*_args, **_kwargs):
         transaction = current_transaction()
         if transaction is None:
-            response = yield from coro
+            response = await coro
             return response
 
         # Patch in should_ignore to all notice_error calls
@@ -352,7 +345,7 @@ def _coro(*_args, **_kwargs):
         import aiohttp.web as _web
 
         try:
-            response = yield from coro
+            response = await coro
         except _web.HTTPException as e:
             _nr_process_response(e, transaction)
             raise

newrelic/hooks/framework_sanic.py

@@ -311,4 +311,4 @@ def instrument_sanic_response(module):
 
 def instrument_sanic_touchup_service(module):
     if hasattr(module, "TouchUp") and hasattr(module.TouchUp, "run"):
-        wrap_function_wrapper(module.TouchUp, "run", _nr_wrap_touchup_run)
+        wrap_function_wrapper(module, "TouchUp.run", _nr_wrap_touchup_run)

setup.py

@@ -121,6 +121,7 @@ def build_extension(self, ext):
     "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: Implementation :: CPython",
     "Programming Language :: Python :: Implementation :: PyPy",
     "Topic :: System :: Monitoring",

tests/agent_features/_test_async_coroutine_trace.py

@@ -14,6 +14,7 @@
 
 import asyncio
 import functools
+import sys
 import time
 
 import pytest
@@ -68,6 +69,7 @@ def _test():
     assert full_metrics[metric_key].total_call_time >= 0.1
 
 
+@pytest.mark.skipif(sys.version_info >= (3, 11), reason="Asyncio decorator was removed in Python 3.11+.")
 @pytest.mark.parametrize(
     "trace,metric",
     [

tests/agent_features/test_async_timing.py

@@ -44,17 +44,15 @@ def _validate_total_time_value(wrapped, instance, args, kwargs):
 
 
 @function_trace(name="child")
-@asyncio.coroutine
-def child():
-    yield from asyncio.sleep(0.1)
+async def child():
+    await asyncio.sleep(0.1)
 
 
 @background_task(name="parent")
-@asyncio.coroutine
-def parent(calls):
+async def parent(calls):
     coros = [child() for _ in range(calls)]
-    yield from asyncio.gather(*coros)
-    yield from asyncio.sleep(0.1)
+    await asyncio.gather(*coros)
+    await asyncio.sleep(0.1)
 
 
 @validate_total_time_value_greater_than(0.2)

tests/agent_features/test_coroutine_trace.py

@@ -105,33 +105,30 @@ def test_coroutine_siblings(event_loop):
     # the case if child was a child of child since child is terminal)
 
     @function_trace("child", terminal=True)
-    @asyncio.coroutine
-    def child(wait, event=None):
+    async def child(wait, event=None):
         if event:
             event.set()
-        yield from wait.wait()
+        await wait.wait()
 
-    @asyncio.coroutine
-    def middle():
+    async def middle():
         wait = asyncio.Event()
         started = asyncio.Event()
 
         child_0 = asyncio.ensure_future(child(wait, started))
 
         # Wait for the first child to start
-        yield from started.wait()
+        await started.wait()
 
         child_1 = asyncio.ensure_future(child(wait))
 
         # Allow children to complete
         wait.set()
-        yield from child_1
-        yield from child_0
+        await child_1
+        await child_0
 
     @function_trace("parent")
-    @asyncio.coroutine
-    def parent():
-        yield from asyncio.ensure_future(middle())
+    async def parent():
+        await asyncio.ensure_future(middle())
 
     event_loop.run_until_complete(parent())
 
@@ -465,15 +462,13 @@ def test_trace_outlives_transaction(event_loop):
     running, finish = asyncio.Event(), asyncio.Event()
 
     @function_trace(name="coro")
-    @asyncio.coroutine
-    def _coro():
+    async def _coro():
         running.set()
-        yield from finish.wait()
+        await finish.wait()
 
-    @asyncio.coroutine
-    def parent():
+    async def parent():
         task.append(asyncio.ensure_future(_coro()))
-        yield from running.wait()
+        await running.wait()
 
     @validate_transaction_metrics(
         "test_trace_outlives_transaction",

tests/agent_features/test_coroutine_transaction.py

@@ -39,8 +39,7 @@
 
 def coroutine_test(event_loop, transaction, nr_enabled=True, does_hang=False, call_exit=False, runtime_error=False):
     @transaction
-    @asyncio.coroutine
-    def task():
+    async def task():
         txn = current_transaction()
 
         if not nr_enabled:
@@ -55,15 +54,15 @@ def task():
 
         try:
             if does_hang:
-                yield from loop.create_future()
+                await loop.create_future()  # noqa
             else:
-                yield from asyncio.sleep(0.0)
+                await asyncio.sleep(0.0)
                 if nr_enabled and txn.enabled:
                     # Validate loop time is recorded after suspend
                     assert txn._loop_time > 0.0
         except GeneratorExit:
             if runtime_error:
-                yield from asyncio.sleep(0.0)
+                await asyncio.sleep(0.0)
 
     return task
 
@@ -160,11 +159,10 @@ def test_async_coroutine_throw_cancel(event_loop, num_coroutines, create_test_ta
 
     tasks = [create_test_task(event_loop, transaction) for _ in range(num_coroutines)]
 
-    @asyncio.coroutine
-    def task_c():
+    async def task_c():
         futures = [asyncio.ensure_future(t()) for t in tasks]
 
-        yield from asyncio.sleep(0.0)
+        await asyncio.sleep(0.0)
 
         [f.cancel() for f in futures]
 
@@ -195,8 +193,7 @@ def test_async_coroutine_throw_error(event_loop, num_coroutines, create_test_tas
 
     tasks = [create_test_task(event_loop, transaction) for _ in range(num_coroutines)]
 
-    @asyncio.coroutine
-    def task_c():
+    async def task_c():
         coros = [t() for t in tasks]
 
         for coro in coros:
@@ -232,14 +229,13 @@ def test_async_coroutine_close(event_loop, num_coroutines, create_test_task, tra
 
     tasks = [create_test_task(event_loop, transaction) for _ in range(num_coroutines)]
 
-    @asyncio.coroutine
-    def task_c():
+    async def task_c():
         coros = [t() for t in tasks]
 
         if start_coroutines:
             [asyncio.ensure_future(coro) for coro in coros]
 
-            yield from asyncio.sleep(0.0)
+            await asyncio.sleep(0.0)
 
         [coro.close() for coro in coros]
 
@@ -273,13 +269,12 @@ def test_async_coroutine_close_raises_error(event_loop, num_coroutines, create_t
 
     tasks = [create_test_task(event_loop, transaction, runtime_error=True) for _ in range(num_coroutines)]
 
-    @asyncio.coroutine
-    def task_c():
+    async def task_c():
         coros = [t() for t in tasks]
 
         [c.send(None) for c in coros]
 
-        yield from asyncio.sleep(0.0)
+        await asyncio.sleep(0.0)
 
         for coro in coros:
             with pytest.raises(RuntimeError):
@@ -313,24 +308,21 @@ def test_deferred_async_background_task(event_loop, transaction, metric, argumen
     args, kwargs = arguments("deferred")
 
     @transaction(*args, **kwargs)
-    @asyncio.coroutine
-    def child_task():
-        yield from asyncio.sleep(0)
+    async def child_task():
+        await asyncio.sleep(0)
 
     main_metric = (metric % "main", "")
 
     args, kwargs = arguments("main")
 
     @transaction(*args, **kwargs)
-    @asyncio.coroutine
-    def parent_task():
-        yield from asyncio.sleep(0)
+    async def parent_task():
+        await asyncio.sleep(0)
         return event_loop.create_task(child_task())
 
-    @asyncio.coroutine
-    def test_runner():
-        child = yield from parent_task()
-        yield from child
+    async def test_runner():
+        child = await parent_task()
+        await child
 
     metrics = []
 
@@ -362,18 +354,16 @@ def test_child_transaction_when_parent_is_running(event_loop, transaction, metri
     args, kwargs = arguments("deferred")
 
     @transaction(*args, **kwargs)
-    @asyncio.coroutine
-    def child_task():
-        yield from asyncio.sleep(0)
+    async def child_task():
+        await asyncio.sleep(0)
 
     main_metric = (metric % "main", "")
 
     args, kwargs = arguments("main")
 
     @transaction(*args, **kwargs)
-    @asyncio.coroutine
-    def parent_task():
-        yield from event_loop.create_task(child_task())
+    async def parent_task():
+        await event_loop.create_task(child_task())
 
     metrics = []
 
@@ -405,9 +395,8 @@ def test_nested_coroutine_inside_sync(event_loop, transaction, metric, arguments
     args, kwargs = arguments("child")
 
     @transaction(*args, **kwargs)
-    @asyncio.coroutine
-    def child_task():
-        yield from asyncio.sleep(0)
+    async def child_task():
+        await asyncio.sleep(0)
 
     main_metric = (metric % "main", "")
     args, kwargs = arguments("main")
@@ -443,22 +432,20 @@ def test_nested_coroutine_task_already_active(event_loop, transaction, metric, a
     args, kwargs = arguments("deferred")
 
     @transaction(*args, **kwargs)
-    @asyncio.coroutine
-    def child_task():
-        yield from asyncio.sleep(0)
+    async def child_task():
+        await asyncio.sleep(0)
 
     @function_trace()
-    def child_trace():
-        yield from child_task()
+    async def child_trace():
+        await child_task()
 
     main_metric = (metric % "main", "")
 
     args, kwargs = arguments("main")
 
     @transaction(*args, **kwargs)
-    @asyncio.coroutine
-    def parent_task():
-        yield from event_loop.create_task(child_trace())
+    async def parent_task():
+        await event_loop.create_task(child_trace())
 
     metrics = []