Allow scrubbing magics via nbdev_export --procs

Documentation on the `scrub_magics` processor is included just below its
associated code. None of the existing tutorials or explanations
notebooks seemed appropriate for this small contribution.

The scrub_magics configuration was removed because the processor is no
longer always executed. However, the existing behavior of always
executing the black_format processor is retained, for compatibility
purposes.

    usage: nbdev_export [-h] [--path PATH] [--symlinks] [--file_glob FILE_GLOB] [--file_re FILE_RE] [--folder_re FOLDER_RE] [--skip_file_glob SKIP_FILE_GLOB] [--skip_file_re SKIP_FILE_RE] [--skip_folder_re SKIP_FOLDER_RE] [--procs PROCS]

    Export notebooks in `path` to Python modules

    optional arguments:
    ...
      --procs PROCS                    whitespace delimited tokens of processors to use. (default: black_format)

Thank you for the feedback on 9366c7f @jph00.

Quick and hopefully unimportant question: You mentioned that this is
a "new directive". Should we refer to these as directives instead of as
processors? I didn't want to muddy up the diff with the refactoring.
The terms do seem similar, but I do worry that calling black_format and
scrub_magics export processors might confuse people who can't find them
under the `10_processors.ipynb` notebook. It certainly confused me at
first.

Author: yegeniy

nbdev/_modidx.py

@@ -77,6 +77,7 @@
                               'nbdev.export.ExportModuleProc.begin': ('api/export.html#exportmoduleproc.begin', 'nbdev/export.py'),
                               'nbdev.export.black_format': ('api/export.html#black_format', 'nbdev/export.py'),
                               'nbdev.export.nb_export': ('api/export.html#nb_export', 'nbdev/export.py'),
+                              'nbdev.export.optional_procs': ('api/export.html#optional_procs', 'nbdev/export.py'),
                               'nbdev.export.scrub_magics': ('api/export.html#scrub_magics', 'nbdev/export.py')},
             'nbdev.extract_attachments': {},
             'nbdev.frontmatter': { 'nbdev.frontmatter.FrontmatterProc': ('api/frontmatter.html#frontmatterproc', 'nbdev/frontmatter.py'),

nbdev/config.py

@@ -59,7 +59,6 @@ def _apply_defaults(
     language='English', # Language PyPI classifier
     recursive:bool_arg=True, # Include subfolders in notebook globs?
     black_formatting:bool_arg=False, # Format libraries with black?
-    scrub_magics:bool_arg=False, # Remove magics from exported modules?
     readme_nb='index.ipynb', # Notebook to export as repo readme
     title='%(lib_name)s', # Quarto website title
     allowed_metadata_keys='', # Preserve the list of keys in the main notebook metadata

nbdev/doclinks.py

@@ -127,23 +127,28 @@ def nbglob_cli(
                   skip_file_glob=skip_file_glob, skip_file_re=skip_file_re, skip_folder_re=skip_folder_re)
 
 # %% ../nbs/api/05_doclinks.ipynb 22
+# the import is needed to call getattr on it
+import nbdev.export
+
 @call_parse
 @delegates(nbglob_cli)
 def nbdev_export(
     path:str=None, # Path or filename
+    procs:str="black_format", # whitespace delimited tokens of processors to use.
     **kwargs):
     "Export notebooks in `path` to Python modules"
     if os.environ.get('IN_TEST',0): return
+    if procs: procs = [getattr(nbdev.export, p) for p in procs.split(" ") if p in optional_procs()]
     files = nbglob(path=path, as_path=True, **kwargs).sorted('name')
-    for f in files: nb_export(f)
+    for f in files: nb_export(f, procs=procs)
     add_init(get_config().lib_path)
     _build_modidx()
 
-# %% ../nbs/api/05_doclinks.ipynb 24
+# %% ../nbs/api/05_doclinks.ipynb 25
 import importlib,ast
 from functools import lru_cache
 
-# %% ../nbs/api/05_doclinks.ipynb 25
+# %% ../nbs/api/05_doclinks.ipynb 26
 def _find_mod(mod):
     mp,_,mr = mod.partition('/')
     spec = importlib.util.find_spec(mp)
@@ -166,7 +171,7 @@ def _get_exps(mod):
 
 def _lineno(sym, fname): return _get_exps(fname).get(sym, None) if fname else None
 
-# %% ../nbs/api/05_doclinks.ipynb 27
+# %% ../nbs/api/05_doclinks.ipynb 28
 def _qual_sym(s, settings):
     if not isinstance(s,tuple): return s
     nb,py = s
@@ -181,10 +186,10 @@ def _qual_syms(entries):
     if 'doc_host' not in settings: return entries
     return {'syms': {mod:_qual_mod(d, settings) for mod,d in entries['syms'].items()}, 'settings':settings}
 
-# %% ../nbs/api/05_doclinks.ipynb 28
+# %% ../nbs/api/05_doclinks.ipynb 29
 _re_backticks = re.compile(r'`([^`\s]+)`')
 
-# %% ../nbs/api/05_doclinks.ipynb 29
+# %% ../nbs/api/05_doclinks.ipynb 30
 @lru_cache(None)
 class NbdevLookup:
     "Mapping from symbol names to docs and source URLs"

nbdev/export.py

@@ -1,7 +1,7 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/api/04_export.ipynb.
 
 # %% auto 0
-__all__ = ['ExportModuleProc', 'black_format', 'scrub_magics', 'nb_export']
+__all__ = ['ExportModuleProc', 'black_format', 'scrub_magics', 'optional_procs', 'nb_export']
 
 # %% ../nbs/api/04_export.ipynb 2
 from .config import *
@@ -26,7 +26,7 @@ def _export_(self, cell, exp_to=None):
         self.in_all[ifnone(exp_to, '#')].append(cell)
     _exports_=_export_
 
-# %% ../nbs/api/04_export.ipynb 7
+# %% ../nbs/api/04_export.ipynb 8
 def black_format(cell, # Cell to format
                  force=False): # Turn black formatting on regardless of settings.ini
     "Processor to format code with `black`"
@@ -40,20 +40,27 @@ def black_format(cell, # Cell to format
         try: cell.source = _format_str(cell.source).strip()
         except: pass
 
-# %% ../nbs/api/04_export.ipynb 9
-_magics_pattern = re.compile(r'^\s*(%%|%).*', re.MULTILINE)
+# %% ../nbs/api/04_export.ipynb 10
+# includes the newline, because calling .strip() would affect all cells.
+_magics_pattern = re.compile(r'^\s*(%%|%).*\n?', re.MULTILINE)
 
 def scrub_magics(cell): # Cell to format
     "Processor to remove cell magics from exported code"
     try: cfg = get_config()
     except FileNotFoundError: return
-    if (not cfg.scrub_magics) or cell.cell_type != 'code': return
-    try:
-      if cell.cell_type == 'code': cell.source = _magics_pattern.sub('', cell.source).strip()
+    if cell.cell_type != 'code': return
+    try: cell.source = _magics_pattern.sub('', cell.source)
     except: pass
 
-# %% ../nbs/api/04_export.ipynb 10
-def nb_export(nbname, lib_path=None, procs=[black_format,scrub_magics], debug=False, mod_maker=ModuleMaker, name=None):
+# %% ../nbs/api/04_export.ipynb 13
+import nbdev.export
+def optional_procs():
+    "An explicit list of processors that could be used by `nb_export`"
+    return L([p for p in nbdev.export.__all__
+              if p not in ["nb_export", "ExportModuleProc", "optional_procs"]])
+
+# %% ../nbs/api/04_export.ipynb 16
+def nb_export(nbname, lib_path=None, procs=None, debug=False, mod_maker=ModuleMaker, name=None):
     "Create module(s) from notebook"
     if lib_path is None: lib_path = get_config().lib_path
     exp = ExportModuleProc()

nbs/api/01_config.ipynb

@@ -155,7 +155,6 @@
     "    language='English', # Language PyPI classifier\n",
     "    recursive:bool_arg=True, # Include subfolders in notebook globs?\n",
     "    black_formatting:bool_arg=False, # Format libraries with black?\n",
-    "    scrub_magics:bool_arg=False, # Remove magics from exported modules?\n",
     "    readme_nb='index.ipynb', # Notebook to export as repo readme\n",
     "    title='%(lib_name)s', # Quarto website title\n",
     "    allowed_metadata_keys='', # Preserve the list of keys in the main notebook metadata\n",

nbs/api/04_export.ipynb

@@ -95,6 +95,13 @@
     "assert 'h_n' in exp.in_all['some.thing'][0].source"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Optional export processors"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -122,7 +129,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "_cell = read_nb('../../tests/black.ipynb')['cells'][0]\n",
+    "_cell = read_nb('../../tests/export_procs.ipynb')['cells'][0]\n",
     "black_format(_cell, force=True)\n",
     "test_eq(_cell.source, 'j = [1, 2, 3]')"
    ]
@@ -134,26 +141,99 @@
    "outputs": [],
    "source": [
     "#|export\n",
-    "_magics_pattern = re.compile(r'^\\s*(%%|%).*', re.MULTILINE)\n",
+    "# includes the newline, because calling .strip() would affect all cells.\n",
+    "_magics_pattern = re.compile(r'^\\s*(%%|%).*\\n?', re.MULTILINE)\n",
     "\n",
     "def scrub_magics(cell): # Cell to format\n",
     "    \"Processor to remove cell magics from exported code\"\n",
     "    try: cfg = get_config()\n",
     "    except FileNotFoundError: return\n",
-    "    if (not cfg.scrub_magics) or cell.cell_type != 'code': return\n",
-    "    try:\n",
-    "      if cell.cell_type == 'code': cell.source = _magics_pattern.sub('', cell.source).strip()\n",
+    "    if cell.cell_type != 'code': return\n",
+    "    try: cell.source = _magics_pattern.sub('', cell.source)\n",
     "    except: pass"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "`scrub_magics` is a processor that scrubs the jupyter \"magics\" lines out of exported cells. This can be helpful when using tools like [sparkmagic](https://github.com/jupyter-incubator/sparkmagic) or just [Jupyter's builtin magics](https://ipython.readthedocs.io/en/stable/interactive/magics.html) in an nbdev project.\n",
+    "\n",
+    "Usage:  \n",
+    "This behavior can be enabled by passing `scrub_magics` into the `--procs` flag of the `nbdev_export` command.\n",
+    "- `nbdev_export --procs scrub_magics`\n",
+    "- `nbdev_export --procs 'scrub_magics black_format'`\n",
+    "\n",
+    "Example:\n",
+    "\n",
+    "A cell like below could export the line `\"hello nbdev\"` into the `bar` module. And the `%%spark` magic line would be omitted.\n",
+    "\n",
+    "```python\n",
+    "%%spark\n",
+    "#|export bar\n",
+    "\"hello nbdev\"\n",
+    "```\n",
+    "\n",
+    "It will export as something similar to this:\n",
+    "\n",
+    "```python\n",
+    "# %% ../path/to/01_bar.ipynb 1\n",
+    "\"hello nbdev\"\n",
+    "```\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "_cell = read_nb('../../tests/export_procs.ipynb')['cells'][2]\n",
+    "scrub_magics(_cell)\n",
+    "test_eq(_cell.source, '''#|export bar\n",
+    "\"hello nbdev\"''')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#|export\n",
+    "import nbdev.export\n",
+    "def optional_procs():\n",
+    "    \"An explicit list of processors that could be used by `nb_export`\"\n",
+    "    return L([p for p in nbdev.export.__all__\n",
+    "              if p not in [\"nb_export\", \"ExportModuleProc\", \"optional_procs\"]])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# every optional processor should be explicitly listed here\n",
+    "test_eq(optional_procs(), ['black_format', 'scrub_magics'])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### `nb_export`"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
     "#|export\n",
-    "def nb_export(nbname, lib_path=None, procs=[black_format,scrub_magics], debug=False, mod_maker=ModuleMaker, name=None):\n",
+    "def nb_export(nbname, lib_path=None, procs=None, debug=False, mod_maker=ModuleMaker, name=None):\n",
     "    \"Create module(s) from notebook\"\n",
     "    if lib_path is None: lib_path = get_config().lib_path\n",
     "    exp = ExportModuleProc()\n",

nbs/api/05_doclinks.ipynb

@@ -347,19 +347,34 @@
    "outputs": [],
    "source": [
     "#|export\n",
+    "# the import is needed to call getattr on it\n",
+    "import nbdev.export\n",
+    "\n",
     "@call_parse\n",
     "@delegates(nbglob_cli)\n",
     "def nbdev_export(\n",
     "    path:str=None, # Path or filename\n",
+    "    procs:str=\"black_format\", # whitespace delimited tokens of processors to use.\n",
     "    **kwargs):\n",
     "    \"Export notebooks in `path` to Python modules\"\n",
     "    if os.environ.get('IN_TEST',0): return\n",
+    "    if procs: procs = [getattr(nbdev.export, p) for p in procs.split(\" \") if p in optional_procs()]\n",
     "    files = nbglob(path=path, as_path=True, **kwargs).sorted('name')\n",
-    "    for f in files: nb_export(f)\n",
+    "    for f in files: nb_export(f, procs=procs)\n",
     "    add_init(get_config().lib_path)\n",
     "    _build_modidx()"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "`procs` takes the names of any optional processors you with to run on the exported cells of your notebook.\n",
+    "It can be a list of processors\n",
+    "\n",
+    "N.B.: the `black_format` processor is passed in by default. But it is a no-op, unless `black_formatting=True` is set in your `settings.ini` configuration."
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -556,7 +571,7 @@
       "text/markdown": [
        "---\n",
        "\n",
-       "[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L211){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
+       "[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L216){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
        "\n",
        "### NbdevLookup.doc\n",
        "\n",
@@ -567,7 +582,7 @@
       "text/plain": [
        "---\n",
        "\n",
-       "[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L211){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
+       "[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L216){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
        "\n",
        "### NbdevLookup.doc\n",
        "\n",
@@ -650,7 +665,7 @@
       "text/markdown": [
        "---\n",
        "\n",
-       "[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L216){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
+       "[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L221){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
        "\n",
        "### NbdevLookup.code\n",
        "\n",
@@ -661,7 +676,7 @@
       "text/plain": [
        "---\n",
        "\n",
-       "[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L216){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
+       "[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L221){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
        "\n",
        "### NbdevLookup.code\n",
        "\n",
@@ -709,7 +724,7 @@
       "text/markdown": [
        "---\n",
        "\n",
-       "[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L233){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
+       "[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L238){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
        "\n",
        "### NbdevLookup.linkify\n",
        "\n",
@@ -718,7 +733,7 @@
       "text/plain": [
        "---\n",
        "\n",
-       "[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L233){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
+       "[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L238){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
        "\n",
        "### NbdevLookup.linkify\n",
        "\n",

tests/black.ipynb tests/export_procs.ipynbtests/black.ipynb renamed to tests/export_procs.ipynb

@@ -12,6 +12,26 @@
     "     3\n",
     "]"
    ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "40855489-6543-4f63-81c4-1127f4e09c31",
+   "metadata": {},
+   "source": [
+    "to test `scrub_magics`:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "22aa79e5-80ba-4b11-a655-ad8d830fa2ef",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%spark\n",
+    "#|export bar\n",
+    "\"hello nbdev\""
+   ]
   }
  ],
  "metadata": {