Skip to content

Commit 923f2b0

Browse files
jph00jph00
authored
Merge pull request #1250 from yegeniy/scrub-magics-squash
exported modules can scrub_magics
2 parents 744a52d + 0f4d622 commit 923f2b0

File tree

6 files changed

+178
-20
lines changed

6 files changed

+178
-20
lines changed

‎nbdev/_modidx.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,9 @@
7676
'nbdev.export.ExportModuleProc._exporti_': ('api/export.html#exportmoduleproc._exporti_', 'nbdev/export.py'),
7777
'nbdev.export.ExportModuleProc.begin': ('api/export.html#exportmoduleproc.begin', 'nbdev/export.py'),
7878
'nbdev.export.black_format': ('api/export.html#black_format', 'nbdev/export.py'),
79-
'nbdev.export.nb_export': ('api/export.html#nb_export', 'nbdev/export.py')},
79+
'nbdev.export.nb_export': ('api/export.html#nb_export', 'nbdev/export.py'),
80+
'nbdev.export.optional_procs': ('api/export.html#optional_procs', 'nbdev/export.py'),
81+
'nbdev.export.scrub_magics': ('api/export.html#scrub_magics', 'nbdev/export.py')},
8082
'nbdev.extract_attachments': {},
8183
'nbdev.frontmatter': { 'nbdev.frontmatter.FrontmatterProc': ('api/frontmatter.html#frontmatterproc', 'nbdev/frontmatter.py'),
8284
'nbdev.frontmatter.FrontmatterProc._update': ( 'api/frontmatter.html#frontmatterproc._update',

‎nbdev/doclinks.py‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -131,19 +131,23 @@ def nbglob_cli(
131131
@delegates(nbglob_cli)
132132
def nbdev_export(
133133
path:str=None, # Path or filename
134+
procs:Param("tokens naming the export processors to use.", nargs="*", choices=optional_procs())="black_format",
134135
**kwargs):
135136
"Export notebooks in `path` to Python modules"
136137
if os.environ.get('IN_TEST',0): return
138+
if procs:
139+
import nbdev.export
140+
procs = [getattr(nbdev.export, p) for p in L(procs)]
137141
files = nbglob(path=path, as_path=True, **kwargs).sorted('name')
138-
for f in files: nb_export(f)
142+
for f in files: nb_export(f, procs=procs)
139143
add_init(get_config().lib_path)
140144
_build_modidx()
141145

142-
# %% ../nbs/api/05_doclinks.ipynb 24
146+
# %% ../nbs/api/05_doclinks.ipynb 25
143147
import importlib,ast
144148
from functools import lru_cache
145149

146-
# %% ../nbs/api/05_doclinks.ipynb 25
150+
# %% ../nbs/api/05_doclinks.ipynb 26
147151
def _find_mod(mod):
148152
mp,_,mr = mod.partition('/')
149153
spec = importlib.util.find_spec(mp)
@@ -166,7 +170,7 @@ def _get_exps(mod):
166170

167171
def _lineno(sym, fname): return _get_exps(fname).get(sym, None) if fname else None
168172

169-
# %% ../nbs/api/05_doclinks.ipynb 27
173+
# %% ../nbs/api/05_doclinks.ipynb 28
170174
def _qual_sym(s, settings):
171175
if not isinstance(s,tuple): return s
172176
nb,py = s
@@ -181,10 +185,10 @@ def _qual_syms(entries):
181185
if 'doc_host' not in settings: return entries
182186
return {'syms': {mod:_qual_mod(d, settings) for mod,d in entries['syms'].items()}, 'settings':settings}
183187

184-
# %% ../nbs/api/05_doclinks.ipynb 28
188+
# %% ../nbs/api/05_doclinks.ipynb 29
185189
_re_backticks = re.compile(r'`([^`\s]+)`')
186190

187-
# %% ../nbs/api/05_doclinks.ipynb 29
191+
# %% ../nbs/api/05_doclinks.ipynb 30
188192
@lru_cache(None)
189193
class NbdevLookup:
190194
"Mapping from symbol names to docs and source URLs"

‎nbdev/export.py‎

Lines changed: 23 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/api/04_export.ipynb.
22

33
# %% auto 0
4-
__all__ = ['ExportModuleProc', 'black_format', 'nb_export']
4+
__all__ = ['ExportModuleProc', 'black_format', 'scrub_magics', 'optional_procs', 'nb_export']
55

66
# %% ../nbs/api/04_export.ipynb 2
77
from .config import *
@@ -26,7 +26,7 @@ def _export_(self, cell, exp_to=None):
2626
self.in_all[ifnone(exp_to, '#')].append(cell)
2727
_exports_=_export_
2828

29-
# %% ../nbs/api/04_export.ipynb 7
29+
# %% ../nbs/api/04_export.ipynb 8
3030
def black_format(cell, # Cell to format
3131
force=False): # Turn black formatting on regardless of settings.ini
3232
"Processor to format code with `black`"
@@ -40,8 +40,27 @@ def black_format(cell, # Cell to format
4040
try: cell.source = _format_str(cell.source).strip()
4141
except: pass
4242

43-
# %% ../nbs/api/04_export.ipynb 9
44-
def nb_export(nbname, lib_path=None, procs=black_format, debug=False, mod_maker=ModuleMaker, name=None):
43+
# %% ../nbs/api/04_export.ipynb 10
44+
# includes the newline, because calling .strip() would affect all cells.
45+
_magics_pattern = re.compile(r'^\s*(%%|%).*\n?', re.MULTILINE)
46+
47+
def scrub_magics(cell): # Cell to format
48+
"Processor to remove cell magics from exported code"
49+
try: cfg = get_config()
50+
except FileNotFoundError: return
51+
if cell.cell_type != 'code': return
52+
try: cell.source = _magics_pattern.sub('', cell.source)
53+
except: pass
54+
55+
# %% ../nbs/api/04_export.ipynb 13
56+
import nbdev.export
57+
def optional_procs():
58+
"An explicit list of processors that could be used by `nb_export`"
59+
return L([p for p in nbdev.export.__all__
60+
if p not in ["nb_export", "ExportModuleProc", "optional_procs"]])
61+
62+
# %% ../nbs/api/04_export.ipynb 16
63+
def nb_export(nbname, lib_path=None, procs=None, debug=False, mod_maker=ModuleMaker, name=None):
4564
"Create module(s) from notebook"
4665
if lib_path is None: lib_path = get_config().lib_path
4766
exp = ExportModuleProc()

‎nbs/api/04_export.ipynb‎

Lines changed: 101 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,13 @@
9595
"assert 'h_n' in exp.in_all['some.thing'][0].source"
9696
]
9797
},
98+
{
99+
"cell_type": "markdown",
100+
"metadata": {},
101+
"source": [
102+
"### Optional export processors"
103+
]
104+
},
98105
{
99106
"cell_type": "code",
100107
"execution_count": null,
@@ -122,7 +129,7 @@
122129
"metadata": {},
123130
"outputs": [],
124131
"source": [
125-
"_cell = read_nb('../../tests/black.ipynb')['cells'][0]\n",
132+
"_cell = read_nb('../../tests/export_procs.ipynb')['cells'][0]\n",
126133
"black_format(_cell, force=True)\n",
127134
"test_eq(_cell.source, 'j = [1, 2, 3]')"
128135
]
@@ -134,7 +141,99 @@
134141
"outputs": [],
135142
"source": [
136143
"#|export\n",
137-
"def nb_export(nbname, lib_path=None, procs=black_format, debug=False, mod_maker=ModuleMaker, name=None):\n",
144+
"# includes the newline, because calling .strip() would affect all cells.\n",
145+
"_magics_pattern = re.compile(r'^\\s*(%%|%).*\\n?', re.MULTILINE)\n",
146+
"\n",
147+
"def scrub_magics(cell): # Cell to format\n",
148+
" \"Processor to remove cell magics from exported code\"\n",
149+
" try: cfg = get_config()\n",
150+
" except FileNotFoundError: return\n",
151+
" if cell.cell_type != 'code': return\n",
152+
" try: cell.source = _magics_pattern.sub('', cell.source)\n",
153+
" except: pass"
154+
]
155+
},
156+
{
157+
"cell_type": "markdown",
158+
"metadata": {},
159+
"source": [
160+
"`scrub_magics` is a processor that scrubs the jupyter \"magics\" lines out of exported cells. This can be helpful when using tools like [sparkmagic](https://github.com/jupyter-incubator/sparkmagic) or just [Jupyter's builtin magics](https://ipython.readthedocs.io/en/stable/interactive/magics.html) in an nbdev project.\n",
161+
"\n",
162+
"Usage: \n",
163+
"This behavior can be enabled by passing `scrub_magics` into the `--procs` flag of the `nbdev_export` command.\n",
164+
"- `nbdev_export --procs scrub_magics`\n",
165+
"- `nbdev_export --procs 'scrub_magics black_format'`\n",
166+
"\n",
167+
"Example:\n",
168+
"\n",
169+
"A cell like below could export the line `\"hello nbdev\"` into the `bar` module. And the `%%spark` magic line would be omitted.\n",
170+
"\n",
171+
"```python\n",
172+
"%%spark\n",
173+
"#|export bar\n",
174+
"\"hello nbdev\"\n",
175+
"```\n",
176+
"\n",
177+
"It will export as something similar to this:\n",
178+
"\n",
179+
"```python\n",
180+
"# %% ../path/to/01_bar.ipynb 1\n",
181+
"\"hello nbdev\"\n",
182+
"```\n",
183+
"\n"
184+
]
185+
},
186+
{
187+
"cell_type": "code",
188+
"execution_count": null,
189+
"metadata": {},
190+
"outputs": [],
191+
"source": [
192+
"_cell = read_nb('../../tests/export_procs.ipynb')['cells'][2]\n",
193+
"scrub_magics(_cell)\n",
194+
"test_eq(_cell.source, '''#|export bar\n",
195+
"\"hello nbdev\"''')"
196+
]
197+
},
198+
{
199+
"cell_type": "code",
200+
"execution_count": null,
201+
"metadata": {},
202+
"outputs": [],
203+
"source": [
204+
"#|export\n",
205+
"import nbdev.export\n",
206+
"def optional_procs():\n",
207+
" \"An explicit list of processors that could be used by `nb_export`\"\n",
208+
" return L([p for p in nbdev.export.__all__\n",
209+
" if p not in [\"nb_export\", \"ExportModuleProc\", \"optional_procs\"]])"
210+
]
211+
},
212+
{
213+
"cell_type": "code",
214+
"execution_count": null,
215+
"metadata": {},
216+
"outputs": [],
217+
"source": [
218+
"# every optional processor should be explicitly listed here\n",
219+
"test_eq(optional_procs(), ['black_format', 'scrub_magics'])"
220+
]
221+
},
222+
{
223+
"cell_type": "markdown",
224+
"metadata": {},
225+
"source": [
226+
"### `nb_export`"
227+
]
228+
},
229+
{
230+
"cell_type": "code",
231+
"execution_count": null,
232+
"metadata": {},
233+
"outputs": [],
234+
"source": [
235+
"#|export\n",
236+
"def nb_export(nbname, lib_path=None, procs=None, debug=False, mod_maker=ModuleMaker, name=None):\n",
138237
" \"Create module(s) from notebook\"\n",
139238
" if lib_path is None: lib_path = get_config().lib_path\n",
140239
" exp = ExportModuleProc()\n",

‎nbs/api/05_doclinks.ipynb‎

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -347,19 +347,33 @@
347347
"outputs": [],
348348
"source": [
349349
"#|export\n",
350+
"\n",
350351
"@call_parse\n",
351352
"@delegates(nbglob_cli)\n",
352353
"def nbdev_export(\n",
353354
" path:str=None, # Path or filename\n",
355+
" procs:Param(\"tokens naming the export processors to use.\", nargs=\"*\", choices=optional_procs())=\"black_format\",\n",
354356
" **kwargs):\n",
355357
" \"Export notebooks in `path` to Python modules\"\n",
356358
" if os.environ.get('IN_TEST',0): return\n",
359+
" if procs:\n",
360+
" import nbdev.export\n",
361+
" procs = [getattr(nbdev.export, p) for p in L(procs)]\n",
357362
" files = nbglob(path=path, as_path=True, **kwargs).sorted('name')\n",
358-
" for f in files: nb_export(f)\n",
363+
" for f in files: nb_export(f, procs=procs)\n",
359364
" add_init(get_config().lib_path)\n",
360365
" _build_modidx()"
361366
]
362367
},
368+
{
369+
"cell_type": "markdown",
370+
"metadata": {},
371+
"source": [
372+
"`procs` names the optional processors you wish to run on the exported cells of your notebook.\n",
373+
"\n",
374+
"N.B.: the `black_format` processor is passed in by default. But it is a no-op, unless `black_formatting=True` is set in your `settings.ini` configuration. You can omit it from `nbdev_export` on the command line by passing in `--procs`."
375+
]
376+
},
363377
{
364378
"cell_type": "markdown",
365379
"metadata": {},
@@ -556,7 +570,7 @@
556570
"text/markdown": [
557571
"---\n",
558572
"\n",
559-
"[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L211){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
573+
"[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L216){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
560574
"\n",
561575
"### NbdevLookup.doc\n",
562576
"\n",
@@ -567,7 +581,7 @@
567581
"text/plain": [
568582
"---\n",
569583
"\n",
570-
"[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L211){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
584+
"[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L216){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
571585
"\n",
572586
"### NbdevLookup.doc\n",
573587
"\n",
@@ -650,7 +664,7 @@
650664
"text/markdown": [
651665
"---\n",
652666
"\n",
653-
"[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L216){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
667+
"[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L221){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
654668
"\n",
655669
"### NbdevLookup.code\n",
656670
"\n",
@@ -661,7 +675,7 @@
661675
"text/plain": [
662676
"---\n",
663677
"\n",
664-
"[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L216){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
678+
"[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L221){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
665679
"\n",
666680
"### NbdevLookup.code\n",
667681
"\n",
@@ -709,7 +723,7 @@
709723
"text/markdown": [
710724
"---\n",
711725
"\n",
712-
"[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L233){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
726+
"[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L238){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
713727
"\n",
714728
"### NbdevLookup.linkify\n",
715729
"\n",
@@ -718,7 +732,7 @@
718732
"text/plain": [
719733
"---\n",
720734
"\n",
721-
"[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L233){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
735+
"[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L238){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
722736
"\n",
723737
"### NbdevLookup.linkify\n",
724738
"\n",

tests/black.ipynb renamed to tests/export_procs.ipynb

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,26 @@
1212
" 3\n",
1313
"]"
1414
]
15+
},
16+
{
17+
"cell_type": "markdown",
18+
"id": "40855489-6543-4f63-81c4-1127f4e09c31",
19+
"metadata": {},
20+
"source": [
21+
"to test `scrub_magics`:"
22+
]
23+
},
24+
{
25+
"cell_type": "code",
26+
"execution_count": null,
27+
"id": "22aa79e5-80ba-4b11-a655-ad8d830fa2ef",
28+
"metadata": {},
29+
"outputs": [],
30+
"source": [
31+
"%%spark\n",
32+
"#|export bar\n",
33+
"\"hello nbdev\""
34+
]
1535
}
1636
],
1737
"metadata": {

0 commit comments

Comments
 (0)