exported modules can scrub_magics via settings.ini

This scrubs the cell magic lines out in the generated code. In some
sense, it is a follow-up to #905.

Would y'all be interested in a PR that adds a new option to the
settings.ini config? It would scrub any `%%magics` from the exported
modules. Fairly similar to how `black_formatting` works today.

---

The specific use-case is to simplify doing exploratory programming in
PySpark. We can now use nbdev notebooks to generate valid [pyspark][0]
code modules. At the same time, the Python-kernel notebook can continue
to use the [`%%spark`][1] magic. We can use a local notebook to interact
with a remote cluster running Livy.

[0]: https://github.com/apache/spark/tree/v3.1.1-rc3/examples/src/main/python
[1]: https://github.com/jupyter-incubator/sparkmagic

---

Please note that the implementation is just copying
`processors.clean_magics`. Not sure how much of an issue that is for
this repository. Should we add a code cell like below above the
scrub_magics method?

```
\#|hide
\# FIXME: reusing processors.clean_magics runs into a circular import
\# from .processors import clean_magics
```

Author: yegeniy

nbdev/_modidx.py

@@ -76,7 +76,8 @@
                               'nbdev.export.ExportModuleProc._exporti_': ('api/export.html#exportmoduleproc._exporti_', 'nbdev/export.py'),
                               'nbdev.export.ExportModuleProc.begin': ('api/export.html#exportmoduleproc.begin', 'nbdev/export.py'),
                               'nbdev.export.black_format': ('api/export.html#black_format', 'nbdev/export.py'),
-                              'nbdev.export.nb_export': ('api/export.html#nb_export', 'nbdev/export.py')},
+                              'nbdev.export.nb_export': ('api/export.html#nb_export', 'nbdev/export.py'),
+                              'nbdev.export.scrub_magics': ('api/export.html#scrub_magics', 'nbdev/export.py')},
             'nbdev.extract_attachments': {},
             'nbdev.frontmatter': { 'nbdev.frontmatter.FrontmatterProc': ('api/frontmatter.html#frontmatterproc', 'nbdev/frontmatter.py'),
                                    'nbdev.frontmatter.FrontmatterProc._update': ( 'api/frontmatter.html#frontmatterproc._update',

nbdev/config.py

@@ -56,6 +56,7 @@ def _apply_defaults(
     language='English', # Language PyPI classifier
     recursive:bool_arg=True, # Include subfolders in notebook globs?
     black_formatting:bool_arg=False, # Format libraries with black?
+    scrub_magics:bool_arg=False, # Remove magics from exported modules?
     readme_nb='index.ipynb', # Notebook to export as repo readme
     title='%(lib_name)s', # Quarto website title
     allowed_metadata_keys='', # Preserve the list of keys in the main notebook metadata

nbdev/export.py

@@ -1,7 +1,7 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/api/export.ipynb.
 
 # %% auto 0
-__all__ = ['ExportModuleProc', 'black_format', 'nb_export']
+__all__ = ['ExportModuleProc', 'black_format', 'scrub_magics', 'nb_export']
 
 # %% ../nbs/api/export.ipynb 2
 from .config import *
@@ -41,7 +41,19 @@ def black_format(cell, # Cell to format
         except: pass
 
 # %% ../nbs/api/export.ipynb 9
-def nb_export(nbname, lib_path=None, procs=black_format, debug=False, mod_maker=ModuleMaker, name=None):
+_magics_pattern = re.compile(r'^\s*(%%|%).*', re.MULTILINE)
+
+def scrub_magics(cell): # Cell to format
+    "Processor to remove cell magics from exported code"
+    try: cfg = get_config()
+    except FileNotFoundError: return
+    if (not cfg.scrub_magics) or cell.cell_type != 'code': return
+    try:
+      if cell.cell_type == 'code': cell.source = _magics_pattern.sub('', cell.source).strip()
+    except: pass
+
+# %% ../nbs/api/export.ipynb 10
+def nb_export(nbname, lib_path=None, procs=[black_format,scrub_magics], debug=False, mod_maker=ModuleMaker, name=None):
     "Create module(s) from notebook"
     if lib_path is None: lib_path = get_config().lib_path
     exp = ExportModuleProc()

nbs/api/config.ipynb

@@ -151,6 +151,7 @@
     "    language='English', # Language PyPI classifier\n",
     "    recursive:bool_arg=True, # Include subfolders in notebook globs?\n",
     "    black_formatting:bool_arg=False, # Format libraries with black?\n",
+    "    scrub_magics:bool_arg=False, # Remove magics from exported modules?\n",
     "    readme_nb='index.ipynb', # Notebook to export as repo readme\n",
     "    title='%(lib_name)s', # Quarto website title\n",
     "    allowed_metadata_keys='', # Preserve the list of keys in the main notebook metadata\n",

nbs/api/export.ipynb

@@ -134,7 +134,26 @@
    "outputs": [],
    "source": [
     "#|export\n",
-    "def nb_export(nbname, lib_path=None, procs=black_format, debug=False, mod_maker=ModuleMaker, name=None):\n",
+    "_magics_pattern = re.compile(r'^\\s*(%%|%).*', re.MULTILINE)\n",
+    "\n",
+    "def scrub_magics(cell): # Cell to format\n",
+    "    \"Processor to remove cell magics from exported code\"\n",
+    "    try: cfg = get_config()\n",
+    "    except FileNotFoundError: return\n",
+    "    if (not cfg.scrub_magics) or cell.cell_type != 'code': return\n",
+    "    try:\n",
+    "      if cell.cell_type == 'code': cell.source = _magics_pattern.sub('', cell.source).strip()\n",
+    "    except: pass"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#|export\n",
+    "def nb_export(nbname, lib_path=None, procs=[black_format,scrub_magics], debug=False, mod_maker=ModuleMaker, name=None):\n",
     "    \"Create module(s) from notebook\"\n",
     "    if lib_path is None: lib_path = get_config().lib_path\n",
     "    exp = ExportModuleProc()\n",