Merge pull request #71 from maartenbreddels/fix_auto_run_nested_with_tornado

fix: always run a coroutine and work with tornado

Author: MSeal

binder/run_nbclient.ipynb

@@ -46,7 +46,7 @@
     "nb = nbf.read('./empty_notebook.ipynb', nbf.NO_CONVERT)\n",
     "\n",
     "# Execute our in-memory notebook, which will now have outputs\n",
-    "nb = nbclient.execute(nb, nest_asyncio=True)"
+    "nb = nbclient.execute(nb)"
    ]
   },
   {

docs/changelog.md

@@ -5,6 +5,7 @@
 ### Major Changes
 
 - Mimic an Output widget at the frontend so that the Output widget behaves correctly [#68](https://github.com/jupyter/nbclient/pull/68)
+- Nested asyncio is automatic, and works with Tornado [#71](https://github.com/jupyter/nbclient/pull/71)
 
 ## 0.3.1
 

nbclient/client.py

@@ -103,21 +103,6 @@ class NotebookClient(LoggingConfigurable):
         ),
     ).tag(config=True)
 
-    nest_asyncio = Bool(
-        False,
-        help=dedent(
-            """
-            If False (default), then blocking functions such as `execute`
-            assume that no event loop is already running. These functions
-            run their async counterparts (e.g. `async_execute`) in an event
-            loop with `asyncio.run_until_complete`, which will fail if an
-            event loop is already running. This can be the case if nbclient
-            is used e.g. in a Jupyter Notebook. In that case, `nest_asyncio`
-            should be set to True.
-            """
-        ),
-    ).tag(config=True)
-
     force_raise_errors = Bool(
         False,
         help=dedent(

nbclient/tests/util.py

@@ -0,0 +1,57 @@
+import asyncio
+
+import tornado
+
+from nbclient.util import run_sync
+
+
+@run_sync
+async def some_async_function():
+    await asyncio.sleep(0.01)
+    return 42
+
+
+def test_nested_asyncio_with_existing_ioloop():
+    ioloop = asyncio.new_event_loop()
+    try:
+        asyncio.set_event_loop(ioloop)
+        assert some_async_function() == 42
+        assert asyncio.get_event_loop() is ioloop
+    finally:
+        asyncio._set_running_loop(None)  # it seems nest_asyncio doesn't reset this
+
+
+def test_nested_asyncio_with_no_ioloop():
+    asyncio.set_event_loop(None)
+    try:
+        assert some_async_function() == 42
+    finally:
+        asyncio._set_running_loop(None)  # it seems nest_asyncio doesn't reset this
+
+
+def test_nested_asyncio_with_tornado():
+    # This tests if tornado accepts the pure-Python Futures, see
+    # https://github.com/tornadoweb/tornado/issues/2753
+    # https://github.com/erdewit/nest_asyncio/issues/23
+    asyncio.set_event_loop(asyncio.new_event_loop())
+    ioloop = tornado.ioloop.IOLoop.current()
+
+    async def some_async_function():
+        future = asyncio.ensure_future(asyncio.sleep(0.1))
+        # this future is a different future after nested-asyncio has patched
+        # the asyncio module, check if tornado likes it:
+        ioloop.add_future(future, lambda f: f.result())
+        await future
+        return 42
+
+    def some_sync_function():
+        return run_sync(some_async_function)()
+
+    async def run():
+        # calling some_async_function directly should work
+        assert await some_async_function() == 42
+        # but via a sync function (using nested-asyncio) can lead to issues:
+        # https://github.com/tornadoweb/tornado/issues/2753
+        assert some_sync_function() == 42
+
+    ioloop.run_sync(run)

nbclient/util.py

@@ -4,9 +4,53 @@
 # Distributed under the terms of the Modified BSD License.
 
 import asyncio
+import sys
 import inspect
 
 
+def check_ipython():
+    # original from vaex/asyncio.py
+    IPython = sys.modules.get('IPython')
+    if IPython:
+        IPython_version = tuple(map(int, IPython.__version__.split('.')))
+        if IPython_version < (7, 0, 0):
+            raise RuntimeError(f'You are using IPython {IPython.__version__} while we require'
+                               '7.0.0+, please update IPython')
+
+
+def check_patch_tornado():
+    """If tornado is imported, add the patched asyncio.Future to its tuple of acceptable Futures"""
+    # original from vaex/asyncio.py
+    if 'tornado' in sys.modules:
+        import tornado.concurrent
+        if asyncio.Future not in tornado.concurrent.FUTURES:
+            tornado.concurrent.FUTURES = tornado.concurrent.FUTURES + (asyncio.Future, )
+
+
+def just_run(coro):
+    """Make the coroutine run, even if there is an event loop running (using nest_asyncio)"""
+    # original from vaex/asyncio.py
+    loop = asyncio._get_running_loop()
+    if loop is None:
+        had_running_loop = False
+        try:
+            loop = asyncio.get_event_loop()
+        except RuntimeError:
+            # we can still get 'There is no current event loop in ...'
+            loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(loop)
+    else:
+        had_running_loop = True
+    if had_running_loop:
+        # if there is a running loop, we patch using nest_asyncio
+        # to have reentrant event loops
+        check_ipython()
+        import nest_asyncio
+        nest_asyncio.apply()
+        check_patch_tornado()
+    return loop.run_until_complete(coro)
+
+
 def run_sync(coro):
     """Runs a coroutine and blocks until it has executed.
 
@@ -24,26 +68,8 @@ def run_sync(coro):
     result :
         Whatever the coroutine returns.
     """
-    def wrapped(self, *args, **kwargs):
-        try:
-            loop = asyncio.get_event_loop()
-        except RuntimeError:
-            loop = asyncio.new_event_loop()
-            asyncio.set_event_loop(loop)
-        if self.nest_asyncio:
-            import nest_asyncio
-            nest_asyncio.apply(loop)
-        try:
-            result = loop.run_until_complete(coro(self, *args, **kwargs))
-        except RuntimeError as e:
-            if str(e) == 'This event loop is already running':
-                raise RuntimeError(
-                    'You are trying to run nbclient in an environment where an '
-                    'event loop is already running. Please pass `nest_asyncio=True` in '
-                    '`NotebookClient.execute` and such methods.'
-                ) from e
-            raise
-        return result
+    def wrapped(*args, **kwargs):
+        return just_run(coro(*args, **kwargs))
     wrapped.__doc__ = coro.__doc__
     return wrapped