Add --verbose flag to exec_nb for real-time output

- Added verbose parameter throughout the execution chain
- Modified CaptureShell._run to conditionally capture stdout/stderr
- When --verbose is used, output streams to terminal in real-time
- Updated documentation in README.md and help text

Author: radekosmulski

README.md

@@ -64,6 +64,7 @@ You can also execute notebooks from the command line with
 
     usage: exec_nb [-h] [--dest DEST] [--exc_stop] [--inject_code INJECT_CODE]
                    [--inject_path INJECT_PATH] [--inject_idx INJECT_IDX]
+                   [--verbose]
                    src
 
     Execute notebook from `src` and save with outputs to `dest`
@@ -78,3 +79,4 @@ You can also execute notebooks from the command line with
       --inject_code INJECT_CODE  Code to inject into a cell
       --inject_path INJECT_PATH  Path to file containing code to inject into a cell
       --inject_idx INJECT_IDX    Cell to replace with `inject_code` (default: 0)
+      --verbose                  Show stdout/stderr during execution (default: False)

execnb/shell.py

@@ -67,10 +67,13 @@ def __init__(self, path:str|Path=None, mpl_format='retina', history=False, timeo
             self._run(f"set_matplotlib_formats('{mpl_format}')")
 
     def _run(self, raw_cell, store_history=False, silent=False, shell_futures=True, cell_id=None,
-                 stdout=True, stderr=True, display=True):
+                 stdout=True, stderr=True, display=True, verbose=False):
         # TODO what if there's a comment?
         semic = raw_cell.rstrip().endswith(';')
-        with capture_output(display=display, stdout=stdout, stderr=stderr) as c:
+        # When verbose is True, don't capture stdout/stderr to allow real-time output
+        capture_stdout = stdout and not verbose
+        capture_stderr = stderr and not verbose
+        with capture_output(display=display, stdout=capture_stdout, stderr=capture_stderr) as c:
             result = super().run_cell(raw_cell, store_history, silent, shell_futures=shell_futures, cell_id=cell_id)
         return AttrDict(result=result, stdout='' if semic else c.stdout, stderr=c.stderr,
                         display_objects=c.outputs, exception=result.error_in_exec, quiet=semic)
@@ -86,14 +89,14 @@ def enable_gui(self, gui=None): pass
 # %% ../nbs/02_shell.ipynb
 @patch
 def run_cell(self:CaptureShell, raw_cell, store_history=False, silent=False, shell_futures=True, cell_id=None,
-             stdout=True, stderr=True, display=True, timeout=None):
+             stdout=True, stderr=True, display=True, timeout=None, verbose=False):
     if not timeout: timeout = self.timeout
     if timeout:
         def handler(*args): raise TimeoutError()
         signal.signal(signal.SIGALRM, handler)
         signal.alarm(timeout)
     try: return self._run(raw_cell, store_history, silent, shell_futures, cell_id=cell_id,
-                          stdout=stdout, stderr=stderr, display=display)
+                          stdout=stdout, stderr=stderr, display=display, verbose=verbose)
     finally:
         if timeout: signal.alarm(0)
 
@@ -144,9 +147,10 @@ def run(self:CaptureShell,
         code:str, # Python/IPython code to run
         stdout=True, # Capture stdout and save as output?
         stderr=True, # Capture stderr and save as output?
-       timeout:Optional[int]=None): # Shell command will time out after {timeout} seconds
+        timeout:Optional[int]=None, # Shell command will time out after {timeout} seconds
+        verbose:bool=False): # Show stdout/stderr during execution
     "Run `code`, returning a list of all outputs in Jupyter notebook format"
-    res = self.run_cell(code, stdout=stdout, stderr=stderr, timeout=timeout)
+    res = self.run_cell(code, stdout=stdout, stderr=stderr, timeout=timeout, verbose=verbose)
     self.result = res.result.result
     self.exc = res.exception
     return _out_nb(res, self.display_formatter)
@@ -157,8 +161,9 @@ async def run_async(self:CaptureShell,
         code: str,  # Python/IPython code to run
         stdout=True,  # Capture stdout and save as output?
         stderr=True,  # Capture stderr and save as output?
-        timeout:Optional[int]=None): # Shell command will time out after {timeout} seconds
-    return self.run(code, stdout=stdout, stderr=stderr, timeout=timeout)
+        timeout:Optional[int]=None, # Shell command will time out after {timeout} seconds
+        verbose:bool=False): # Show stdout/stderr during execution
+    return self.run(code, stdout=stdout, stderr=stderr, timeout=timeout, verbose=verbose)
 
 # %% ../nbs/02_shell.ipynb
 def _pre(s, xtra=''): return f"<pre {xtra}><code>{escape(s)}</code></pre>"
@@ -197,11 +202,11 @@ def render_output(out):
 
 # %% ../nbs/02_shell.ipynb
 @patch
-def cell(self:CaptureShell, cell, stdout=True, stderr=True):
+def cell(self:CaptureShell, cell, stdout=True, stderr=True, verbose=False):
     "Run `cell`, skipping if not code, and store outputs back in cell"
     if cell.cell_type!='code': return
     self._cell_idx = cell.idx_ + 1
-    outs = self.run(cell.source)
+    outs = self.run(cell.source, verbose=verbose)
     if outs: cell.outputs = _dict2obj(outs)
 
 # %% ../nbs/02_shell.ipynb
@@ -239,13 +244,14 @@ def run_all(self:CaptureShell,
             preproc:callable=_false, # Called before each cell is executed
             postproc:callable=_false, # Called after each cell is executed
             inject_code:str|None=None, # Code to inject into a cell
-            inject_idx:int=0 # Cell to replace with `inject_code`
+            inject_idx:int=0, # Cell to replace with `inject_code`
+            verbose:bool=False # Show stdout/stderr during execution
            ):
     "Run all cells in `nb`, stopping at first exception if `exc_stop`"
     if inject_code is not None: nb.cells[inject_idx].source = inject_code
     for cell in nb.cells:
         if not preproc(cell):
-            self.cell(cell)
+            self.cell(cell, verbose=verbose)
             postproc(cell)
         if self.exc and exc_stop: raise self.exc from None
 
@@ -259,15 +265,16 @@ def execute(self:CaptureShell,
             postproc:callable=_false, # Called after each cell is executed
             inject_code:str|None=None, # Code to inject into a cell
             inject_path:str|Path|None=None, # Path to file containing code to inject into a cell
-            inject_idx:int=0 # Cell to replace with `inject_code`
+            inject_idx:int=0, # Cell to replace with `inject_code`
+            verbose:bool=False # Show stdout/stderr during execution
 ):
     "Execute notebook from `src` and save with outputs to `dest"
     nb = read_nb(src)
     self._fname = src
     self.set_path(Path(src).parent.resolve())
     if inject_path is not None: inject_code = Path(inject_path).read_text()
     self.run_all(nb, exc_stop=exc_stop, preproc=preproc, postproc=postproc,
-                 inject_code=inject_code, inject_idx=inject_idx)
+                 inject_code=inject_code, inject_idx=inject_idx, verbose=verbose)
     if dest: write_nb(nb, dest)
 
 # %% ../nbs/02_shell.ipynb
@@ -290,11 +297,12 @@ def exec_nb(
     exc_stop:bool=False, # Stop on exceptions?
     inject_code:str=None, # Code to inject into a cell
     inject_path:str=None, # Path to file containing code to inject into a cell
-    inject_idx:int=0 # Cell to replace with `inject_code`
+    inject_idx:int=0, # Cell to replace with `inject_code`
+    verbose:bool=False # Show stdout/stderr during execution
 ):
     "Execute notebook from `src` and save with outputs to `dest`"
     CaptureShell().execute(src, dest, exc_stop=exc_stop, inject_code=inject_code,
-                           inject_path=inject_path, inject_idx=inject_idx)
+                           inject_path=inject_path, inject_idx=inject_idx, verbose=verbose)
 
 # %% ../nbs/02_shell.ipynb
 class SmartCompleter(IPCompleter):