fix issues for fastai

Author: jph00

nbdev/__init__.py

@@ -1,3 +1,5 @@
 __version__ = "2.0.1.dev3"
 
 from .doclinks import nbdev_export
+from .showdoc import show_doc
+

nbdev/_modidx.py

@@ -146,6 +146,7 @@
                                   'nbdev.processors.is_frontmatter': 'https://nbdev.fast.ai/processors#is_frontmatter',
                                   'nbdev.processors.lang_identify': 'https://nbdev.fast.ai/processors#lang_identify',
                                   'nbdev.processors.nb_fmdict': 'https://nbdev.fast.ai/processors#nb_fmdict',
+                                  'nbdev.processors.nbflags_': 'https://nbdev.fast.ai/processors#nbflags_',
                                   'nbdev.processors.populate_language': 'https://nbdev.fast.ai/processors#populate_language',
                                   'nbdev.processors.rm_export': 'https://nbdev.fast.ai/processors#rm_export',
                                   'nbdev.processors.rm_header_dash': 'https://nbdev.fast.ai/processors#rm_header_dash',

nbdev/cli.py

@@ -86,7 +86,7 @@ def _nothing(self): return []
     def base_preprocs(self): return [populate_language, infer_frontmatter, add_show_docs, insert_warning]
     def base_postprocs(self): return []
     def base_procs(self):
-        return [lang_identify, strip_ansi, hide_line, filter_stream_, rm_header_dash,
+        return [nbflags_, lang_identify, strip_ansi, hide_line, filter_stream_, rm_header_dash,
                 clean_show_doc, exec_show_docs, rm_export, clean_magics, hide_, add_links,
                strip_hidden_metadata]
 

nbdev/doclinks.py

@@ -52,9 +52,9 @@ def write_nbdev_idx(self:DocLinks):
 # %% ../nbs/04b_doclinks.ipynb 15
 def _binop_leafs(bo, o):
     if isinstance(bo.left, ast.BinOp): left = _binop_leafs(bo.left, o)
-    else: left = [f'{bo.left.id}.{o.name}']
+    else: left = [f'{_id_or_attr(bo.left)}.{o.name}']
     if isinstance(bo.right, ast.BinOp): right = _binop_leafs(bo.right, o)
-    else: right = [f'{bo.right.id}.{o.name}']
+    else: right = [f'{_id_or_attr(bo.right)}.{o.name}']
     return concat(left + right)
 
 # %% ../nbs/04b_doclinks.ipynb 16
@@ -68,14 +68,18 @@ def _get_patch(o):
     if not isinstance(o, (ast.FunctionDef,ast.AsyncFunctionDef)): return
     return first([d for d in o.decorator_list if decor_id(d).startswith('patch')])
 
+def _id_or_attr(a):
+    if hasattr(a, 'id'): return a.id
+    return a.attr
+
 def get_patch_name(o):
     d = _get_patch(o)
     if not d: return
     nm = decor_id(d)
     if nm=='patch': 
         a = o.args.args[0].annotation
         if isinstance(a, ast.BinOp): return _binop_leafs(a, o)
-        else: pre = a.id        
+        else: pre = _id_or_attr(a)
     elif nm=='patch_to': pre = o.decorator_list[0].args[0].id
     else: return
     return f'{pre}.{o.name}'
@@ -140,7 +144,7 @@ def nbdev_export(
     file_re:str=None, # Only include files matching regex
     folder_re:str=None, # Only enter folders matching regex
     skip_file_glob:str=None, # Skip files matching glob
-    skip_file_re:str=None # Skip files matching regex
+    skip_file_re:str='^[_.]' # Skip files matching regex
 ):
     "Export notebooks in `path` to Python modules"
     if os.environ.get('IN_TEST',0): return

nbdev/process.py

@@ -128,19 +128,30 @@ class NBProcessor:
     def __init__(self, path=None, procs=None, preprocs=None, postprocs=None, nb=None, debug=False, rm_directives=True, process=False):
         self.nb = read_nb(path) if nb is None else nb
         self.lang = nb_lang(self.nb)
+        for cell in self.nb.cells: cell.directives_ = extract_directives(cell, remove=rm_directives, lang=self.lang)
         self.procs,self.preprocs,self.postprocs = map_ex((procs,preprocs,postprocs), _mk_procs, nb=self.nb)
         self.debug,self.rm_directives = debug,rm_directives
         if process: self.process()
 
     def _process_cell(self, cell):
         self.cell = cell
+        cell.nb = self.nb
         for proc in self.procs:
+            if not hasattr(cell,'source'): return
+            if not hasattr(cell, 'directives_'):
+                cell.nb=None
+                raise Exception(cell)
             if cell.cell_type=='code' and cell.directives_:
                 for cmd,args in cell.directives_.items():
                     self._process_comment(proc, cell, cmd, args)
                     if not hasattr(cell,'source'): return
-            if callable(proc) and not _is_direc(proc): cell = opt_set(cell, proc(cell))
-            if not hasattr(cell,'source'): return
+            if callable(proc) and not _is_direc(proc):
+                cell = opt_set(cell, proc(cell))
+#                 try: cell = opt_set(cell, proc(cell))
+#                 except:
+#                     cell.nb = None
+#                     raise Exception((proc,cell))
+        cell.nb = None
 
     def _process_comment(self, proc, cell, cmd, args):
         if _is_direc(proc) and getattr(proc, '__name__', '-')[:-1]==cmd: f=proc
@@ -154,7 +165,6 @@ def process(self):
         for proc in self.preprocs:
             self.nb = opt_set(self.nb, proc(self.nb))
             for i,cell in enumerate(self.nb.cells): cell.idx_ = i
-        for cell in self.nb.cells: cell.directives_ = extract_directives(cell, remove=self.rm_directives, lang=self.lang)
         for cell in self.nb.cells: self._process_cell(cell)
         for proc in self.postprocs: self.nb = opt_set(self.nb, proc(self.nb))
         self.nb.cells = [c for c in self.nb.cells if c and getattr(c,'source',None) is not None]

nbdev/processors.py

@@ -1,7 +1,7 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/09_processors.ipynb.
 
 # %% auto 0
-__all__ = ['DEFAULT_FM_KEYS', 'cell_lang', 'add_links', 'strip_ansi', 'strip_hidden_metadata', 'hide_', 'hide_line',
+__all__ = ['DEFAULT_FM_KEYS', 'nbflags_', 'cell_lang', 'add_links', 'strip_ansi', 'strip_hidden_metadata', 'hide_', 'hide_line',
            'filter_stream_', 'clean_magics', 'lang_identify', 'rm_header_dash', 'rm_export', 'clean_show_doc',
            'exec_show_docs', 'populate_language', 'insert_warning', 'add_show_docs', 'is_frontmatter', 'nb_fmdict',
            'construct_fm', 'insert_frontmatter', 'infer_frontmatter']
@@ -22,6 +22,11 @@
 import sys
 
 # %% ../nbs/09_processors.ipynb 9
+def nbflags_(nbp, cell, *args):
+    "Hide cell from output"
+    nbp.nb._nbflags = args
+
+# %% ../nbs/09_processors.ipynb 11
 def cell_lang(cell): return nested_attr(cell, 'metadata.language', 'python')
 
 def add_links(cell):
@@ -32,25 +37,25 @@ def add_links(cell):
         if hasattr(o, 'data') and hasattr(o['data'], 'text/markdown'):
             o.data['text/markdown'] = [nl.link_line(s) for s in o.data['text/markdown']]
 
-# %% ../nbs/09_processors.ipynb 12
+# %% ../nbs/09_processors.ipynb 14
 _re_ansi_escape = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])')
 
 def strip_ansi(cell):
     "Strip Ansi Characters."
     for outp in cell.get('outputs', []):
         if outp.get('name')=='stdout': outp['text'] = [_re_ansi_escape.sub('', o) for o in outp.text]
 
-# %% ../nbs/09_processors.ipynb 14
+# %% ../nbs/09_processors.ipynb 16
 def strip_hidden_metadata(cell):
     '''Strips "hidden" metadata property from code cells so it doesn't interfere with docs rendering'''
     if cell.cell_type == 'code' and 'metadata' in cell: cell.metadata.pop('hidden',None)
 
-# %% ../nbs/09_processors.ipynb 16
+# %% ../nbs/09_processors.ipynb 17
 def hide_(nbp, cell):
     "Hide cell from output"
     del(cell['source'])
 
-# %% ../nbs/09_processors.ipynb 18
+# %% ../nbs/09_processors.ipynb 19
 def _re_hideline(lang=None): return re.compile(fr'{langs[lang]}\|\s*hide_line\s*$', re.MULTILINE)
 
 def hide_line(cell):
@@ -59,22 +64,22 @@ def hide_line(cell):
     if cell.cell_type == 'code' and _re_hideline(lang).search(cell.source):
         cell.source = '\n'.join([c for c in cell.source.splitlines() if not _re_hideline(lang).search(c)])
 
-# %% ../nbs/09_processors.ipynb 20
+# %% ../nbs/09_processors.ipynb 21
 def filter_stream_(nbp, cell, *words):
     "Remove output lines containing any of `words` in `cell` stream output"
     if not words: return
     for outp in cell.get('outputs', []):
         if outp.output_type == 'stream':
             outp['text'] = [l for l in outp.text if not re.search('|'.join(words), l)]
 
-# %% ../nbs/09_processors.ipynb 22
+# %% ../nbs/09_processors.ipynb 23
 _magics_pattern = re.compile(r'^\s*(%%|%).*', re.MULTILINE)
 
 def clean_magics(cell):
     "A preprocessor to remove cell magic commands"
     if cell.cell_type == 'code': cell.source = _magics_pattern.sub('', cell.source).strip()
 
-# %% ../nbs/09_processors.ipynb 24
+# %% ../nbs/09_processors.ipynb 25
 _langs = 'bash|html|javascript|js|latex|markdown|perl|ruby|sh|svg'
 _lang_pattern = re.compile(rf'^\s*%%\s*({_langs})\s*$', flags=re.MULTILINE)
 
@@ -84,7 +89,7 @@ def lang_identify(cell):
         lang = _lang_pattern.findall(cell.source)
         if lang: cell.metadata.language = lang[0]
 
-# %% ../nbs/09_processors.ipynb 27
+# %% ../nbs/09_processors.ipynb 28
 _re_hdr_dash = re.compile(r'^#+\s+.*\s+-\s*$', re.MULTILINE)
 
 def rm_header_dash(cell):
@@ -93,15 +98,15 @@ def rm_header_dash(cell):
         src = cell.source.strip()
         if cell.cell_type == 'markdown' and src.startswith('#') and src.endswith(' -'): del(cell['source'])
 
-# %% ../nbs/09_processors.ipynb 29
+# %% ../nbs/09_processors.ipynb 30
 _hide_dirs = {'export','exporti', 'hide','default_exp'}
 
 def rm_export(cell):
     "Remove cells that are exported or hidden"
     if cell.directives_:
         if cell.directives_.keys() & _hide_dirs: del(cell['source'])
 
-# %% ../nbs/09_processors.ipynb 31
+# %% ../nbs/09_processors.ipynb 32
 _re_showdoc = re.compile(r'^show_doc', re.MULTILINE)
 def _is_showdoc(cell): return cell['cell_type'] == 'code' and _re_showdoc.search(cell.source)
 
@@ -110,7 +115,7 @@ def clean_show_doc(cell):
     if not _is_showdoc(cell): return
     cell.source = '#| echo: false\n' + cell.source
 
-# %% ../nbs/09_processors.ipynb 32
+# %% ../nbs/09_processors.ipynb 33
 _imps = {ast.Import, ast.ImportFrom}
 
 def _show_docs(trees):
@@ -126,47 +131,50 @@ def _do_eval(cell):
     if cell.directives_.keys() & _show_dirs or filter_ex(trees, risinstance(_imps)): return True
     if _show_docs(trees): return True
 
-# %% ../nbs/09_processors.ipynb 33
+# %% ../nbs/09_processors.ipynb 34
 class exec_show_docs:
     "Execute cells needed for `show_docs` output, including exported cells and imports"
     def __init__(self, nb):
         self.k = CaptureShell()
         if nb_lang(nb) == 'python': self.k.run_cell('from nbdev.showdoc import show_doc')
 
     def __call__(self, cell):
+        flags = getattr(cell.nb, '_nbflags', [])
+        if 'skip_showdoc' in flags: return
         if _do_eval(cell): self.k.cell(cell)
         if self.k.exc: 
             sys.stderr.write(f'Error in cell {cell.idx_}:\n{cell.source}')
             raise self.k.exc[1] from None
 
-# %% ../nbs/09_processors.ipynb 36
+# %% ../nbs/09_processors.ipynb 37
 def populate_language(nb):
     "Insert cell language indicator based on notebook metadata.  You should to use this before `lang_identify`"
     for cell in nb.cells:
         if cell.cell_type == 'code': cell.metadata.language = nb_lang(nb)
 
-# %% ../nbs/09_processors.ipynb 39
+# %% ../nbs/09_processors.ipynb 40
 def insert_warning(nb):
     "Insert Autogenerated Warning Into Notebook after the first cell."
     content = "<!-- WARNING: THIS FILE WAS AUTOGENERATED! DO NOT EDIT! -->"
-    nb.cells.insert(1, mk_cell(content, False))
+    nb.cells.insert(1, mk_cell(content, 'markdown'))
 
-# %% ../nbs/09_processors.ipynb 43
+# %% ../nbs/09_processors.ipynb 44
 _def_types = (ast.FunctionDef,ast.AsyncFunctionDef,ast.ClassDef)
 def _def_names(cell, shown):
     return [showdoc_nm(o) for o in concat(cell.parsed_())
             if isinstance(o,_def_types) and o.name not in shown and o.name[0]!='_']
 
-_re_exps = re.compile(r'^\s*#\|\s*(?:export|exports)').search
-
 def _get_nm(tree):
     i = tree.value.args[0]
     return f'{i.value.id}.{i.attr}' if isinstance(i, ast.Attribute) else i.id
 
-# %% ../nbs/09_processors.ipynb 44
+# %% ../nbs/09_processors.ipynb 45
 def add_show_docs(nb):
     "Add show_doc cells after exported cells, unless they are already documented"
-    exports = L(cell for cell in nb.cells if cell.source and _re_exps(cell.source))
+    def _want(c):
+        return c.source and c.cell_type=='code' and ('export' in c.directives_ or 'exports' in c.directives_)
+
+    exports = L(cell for cell in nb.cells if _want(cell))
     trees = nb.cells.map(NbCell.parsed_).concat()
     shown_docs = {_get_nm(t) for t in _show_docs(trees)}
     for cell in reversed(exports):
@@ -175,7 +183,7 @@ def add_show_docs(nb):
         for nm in _def_names(cell, shown_docs):
             nb.cells.insert(cell.idx_+1, mk_cell(f'show_doc({nm})'))
 
-# %% ../nbs/09_processors.ipynb 48
+# %% ../nbs/09_processors.ipynb 49
 _re_title = re.compile(r'^#\s+(.*)[\n\r]+(?:^>\s+(.*))?', flags=re.MULTILINE)
 _re_fm = re.compile(r'^---.*\S+.*---', flags=re.DOTALL)
 _re_defaultexp = re.compile(r'^\s*#\|\s*default_exp\s+(\S+)', flags=re.MULTILINE)
@@ -186,14 +194,14 @@ def _istitle(cell):
     txt = cell.get('source', '')
     return bool(_re_title.search(txt)) if txt else False
 
-# %% ../nbs/09_processors.ipynb 49
+# %% ../nbs/09_processors.ipynb 50
 def _default_exp(nb):
     "get the default_exp from a notebook"
     code_src = nb.cells.filter(lambda x: x.cell_type == 'code').attrgot('source')
     default_exp = first(code_src.filter().map(_re_defaultexp.search).filter())
     return default_exp.group(1) if default_exp else None
 
-# %% ../nbs/09_processors.ipynb 51
+# %% ../nbs/09_processors.ipynb 52
 def nb_fmdict(nb, remove=True): 
     "Infer the front matter from a notebook's markdown formatting"
     md_cells = _celltyp(nb, 'markdown').filter(_istitle)
@@ -209,21 +217,21 @@ def nb_fmdict(nb, remove=True):
         return flags
     else: return {}
 
-# %% ../nbs/09_processors.ipynb 54
+# %% ../nbs/09_processors.ipynb 55
 DEFAULT_FM_KEYS = ['title', 'description', 'author', 'image', 'categories', 'output-file', 'aliases']
 
 def construct_fm(fmdict:dict, keys = DEFAULT_FM_KEYS):
     "construct front matter from a dictionary, but only for `keys`"
     if not fmdict: return None
     return '---\n'+'\n'.join([f"{k}: {fmdict[k]}" for k in keys if k in fmdict])+'\n---'
 
-# %% ../nbs/09_processors.ipynb 56
+# %% ../nbs/09_processors.ipynb 57
 def insert_frontmatter(nb, fm_dict:dict, filter_keys:list=DEFAULT_FM_KEYS):
     "Add frontmatter into notebook based on `filter_keys` that exist in `fmdict`."
     fm = construct_fm(fm_dict, keys=filter_keys)
-    if fm: nb.cells.insert(0, NbCell(0, dict(cell_type='raw', metadata={}, source=fm)))
+    if fm: nb.cells.insert(0, NbCell(0, dict(cell_type='raw', metadata={}, source=fm, directives_={})))
 
-# %% ../nbs/09_processors.ipynb 57
+# %% ../nbs/09_processors.ipynb 58
 def infer_frontmatter(nb):
     "Insert front matter if it doesn't exist automatically from nbdev styled markdown."
     if is_frontmatter(nb): return

nbdev/read.py

@@ -18,9 +18,10 @@
 from pprint import pformat,pprint
 
 # %% ../nbs/01_read.ipynb 4
-def mk_cell(text, code=True):
+def mk_cell(text, cell_type='code'):
     "Create a `NbCell` containing `text`"
-    return NbCell(0, dict(cell_type='code' if code else 'markdown', metadata={}, source=text))
+    assert cell_type in {'code', 'markdown', 'raw'}
+    return NbCell(0, dict(cell_type=cell_type, metadata={}, source=text, directives_={}))
 
 # %% ../nbs/01_read.ipynb 5
 def create_output(txt, mime):