run coroutine under run_sync. Closes #4122 (#4137)

* run coroutine under run_sync. Closes #4122

* update jupyter_core reqs to include run_sync

* upgrade requirements

Author: cscheid

src/resources/jupyter/log.py

@@ -1,6 +1,7 @@
 
 import sys
 import logging
+import os
 
 TRACE = 25
 
@@ -12,6 +13,7 @@ def log_init(log_file, trace = False):
    else:
       logger.setLevel(logging.WARNING)
 
+   global file_handler
    # create handlers
    stderr_handler = logging.StreamHandler(sys.stderr)
    file_handler = logging.FileHandler(log_file)
@@ -25,8 +27,14 @@ def log_init(log_file, trace = False):
    logger.addHandler(stderr_handler)  
    logger.addHandler(file_handler)   
 
-def log(level, msg):
-   logging.getLogger().log(level, msg)
+# force flushing so programs which hang still produce output
+if os.getenv("QUARTO_JUPYTER_FLUSH_LOGS"):
+   def log(level, msg):
+      logging.getLogger().log(level, msg)
+      file_handler.flush()
+else:
+   def log(level, msg):
+      logging.getLogger().log(level, msg)
 
 def log_error(msg, exc_info = False):
    logging.getLogger().log(logging.ERROR, msg, exc_info = exc_info, stack_info = not exc_info)

src/resources/jupyter/notebook.py

@@ -13,8 +13,11 @@
 
 from poyo import parse_string
 
+from log import trace
 import nbformat
 from nbclient import NotebookClient
+from jupyter_client import KernelManager
+from jupyter_core.utils import run_sync
 
 # optional import of papermill for params support
 try:
@@ -37,6 +40,7 @@ class RestartKernel(Exception):
 # execute a notebook
 def notebook_execute(options, status):
 
+   trace('inside notebook_execute')
    # if this is a re-execution of a previously loaded kernel,
    # make sure the underlying python version hasn't changed
    python_cmd = options.get("python_cmd", None)
@@ -97,6 +101,7 @@ def notebook_execute(options, status):
    # read the notebook
    nb = nbformat.read(input, as_version = NB_FORMAT_VERSION)
 
+   trace('read notebook')
    # inject parameters if provided
    if params:
       nb_parameterize(nb, params)
@@ -108,17 +113,21 @@ def notebook_execute(options, status):
    # are we using the cache, if so connect to the cache, and then if we aren't in 'refresh'
    # (forced re-execution) mode then try to satisfy the execution request from the cache
    if cache == True or cache == "refresh":
+      trace('using cache')
       if not get_cache:
           raise ImportError('The jupyter-cache package is required for cached execution')
+      trace('getting cache')
       nb_cache = get_cache(".jupyter_cache")
       if not cache == "refresh":
          cached_nb = nb_from_cache(nb, nb_cache)
          if cached_nb:
             cached_nb.cells.pop(0)
             nb_write(cached_nb, input)
             status("(Notebook read from cache)\n\n")
+            trace('(Notebook read from cache)')
             return True # can persist kernel
    else:
+      trace('not using cache')
       nb_cache = None
 
    # create resources for execution
@@ -130,8 +139,11 @@ def notebook_execute(options, status):
    if run_path:
       resources["metadata"]["path"] = run_path
 
+   trace("Will attempt to create notebook")
    # create NotebookClient
+   trace("type of notebook: {0}".format(type(nb)))
    client, created = notebook_init(nb, resources, allow_errors)
+   trace("NotebookClient created")
 
    # complete progress if necessary
    if (not quiet) and created:
@@ -155,6 +167,7 @@ def notebook_execute(options, status):
 
       # execute cell
       if cell.cell_type == 'code':
+         trace("Executing cell {0}".format(index))
          cell = cell_execute(
             client, 
             cell, 
@@ -164,6 +177,7 @@ def notebook_execute(options, status):
             index > 0 # add_to_history
          )
          cell.execution_count = current_code_cell
+         trace("Executed cell {0}".format(index))
 
       # if this was the setup cell, see if we need to exit b/c dependencies are out of date
       if index == 0:
@@ -195,6 +209,9 @@ def notebook_execute(options, status):
       # end progress
       if progress:
          status("Done\n")
+         trace("Done")
+
+   trace("Notebook execution complete")
 
    # set widgets metadata   
    client.set_widgets_metadata()
@@ -236,14 +253,21 @@ def notebook_init(nb, resources, allow_errors):
    created = False
    if not hasattr(notebook_init, "client"):
 
+      trace("Creating NotebookClient")
       # create notebook client
       client = NotebookClient(nb, resources = resources)
       client.allow_errors = allow_errors
       client.record_timing = False
       client.create_kernel_manager()
       client.start_new_kernel()
       client.start_new_kernel_client()
-      info_msg = client.wait_for_reply(client.kc.kernel_info())
+      info = client.kc.kernel_info()
+
+      async def get_info():
+         return await client.kc.kernel_info()
+      info = run_sync(get_info)()
+
+      info_msg = client.wait_for_reply(info)
       client.nb.metadata['language_info'] = info_msg['content']['language_info']
       notebook_init.client = client
       created = True

tests/requirements.txt

@@ -1,8 +1,10 @@
+anyio==3.6.2
 appnope==0.1.2
 argon2-cffi==20.1.0
 async-generator==1.10
 attrs==21.2.0
 backcall==0.2.0
+beautifulsoup4==4.11.1
 bleach==3.3.0
 bokeh==3.0.3
 cffi==1.14.5
@@ -11,7 +13,9 @@ cycler==0.10.0
 decorator==5.0.9
 defusedxml==0.7.1
 entrypoints==0.3
+fastjsonschema==2.16.2
 fonttools==4.38.0
+idna==3.4
 ipykernel==5.5.5
 ipython==7.25.0
 ipython-genutils==0.2.0
@@ -20,19 +24,23 @@ jedi==0.18.0
 Jinja2==3.0.1
 jsonschema==3.2.0
 jupyter==1.0.0
-jupyter-client==6.1.12
-jupyter-console==6.4.0
-jupyter-core==4.7.1
+jupyter-console==6.4.4
+jupyter-events==0.6.3
+jupyter_client==8.0.1
+jupyter_core==5.1.5
+jupyter_server==2.1.0
+jupyter_server_terminals==0.4.4
 jupyterlab-pygments==0.1.2
 jupyterlab-widgets==1.0.0
 kiwisolver==1.4.4
+lxml==4.9.2
 MarkupSafe==2.0.1
 matplotlib==3.5.3
 matplotlib-inline==0.1.2
 mistune==0.8.4
 nbclient==0.5.3
-nbconvert==6.1.0
-nbformat==5.1.3
+nbconvert==6.5.4
+nbformat==5.7.3
 nest-asyncio==1.5.1
 notebook==6.4.0
 numpy==1.23.2
@@ -43,26 +51,34 @@ parso==0.8.2
 pexpect==4.8.0
 pickleshare==0.7.5
 Pillow==9.2.0
+platformdirs==2.6.2
 prometheus-client==0.11.0
 prompt-toolkit==3.0.19
 ptyprocess==0.7.0
 pycparser==2.20
 Pygments==2.9.0
 pyparsing==3.0.9
 pyrsistent==0.17.3
-python-dateutil==2.8.1
+python-dateutil==2.8.2
+python-json-logger==2.0.4
 pytz==2022.7.1
 PyYAML==6.0
 pyzmq==24.0.1
 qtconsole==5.1.0
 QtPy==1.9.0
+rfc3339-validator==0.1.4
+rfc3986-validator==0.1.1
 Send2Trash==1.7.1
 six==1.16.0
+sniffio==1.3.0
+soupsieve==2.3.2.post1
 terminado==0.10.1
 testpath==0.5.0
-tornado==6.1
-traitlets==5.0.5
+tinycss2==1.2.1
+tornado==6.2
+traitlets==5.8.1
 wcwidth==0.2.5
 webencodings==0.5.1
+websocket-client==1.5.0
 widgetsnbextension==3.5.1
 xyzservices==2022.9.0