fixes #890

Author: jph00

nbdev/_modidx.py

@@ -109,6 +109,10 @@
                               'nbdev.export.nb_export': 'https://nbdev.fast.ai/export.html#nb_export'},
             'nbdev.extract_attachments': { 'nbdev.extract_attachments.ExtractAttachmentsPreprocessor': 'https://nbdev.fast.ai/extract_attachments.html#extractattachmentspreprocessor',
                                            'nbdev.extract_attachments.ExtractAttachmentsPreprocessor.preprocess_cell': 'https://nbdev.fast.ai/extract_attachments.html#extractattachmentspreprocessor.preprocess_cell'},
+            'nbdev.frontmatter': { 'nbdev.frontmatter.FrontmatterProc': 'https://nbdev.fast.ai/frontmatter.html#frontmatterproc',
+                                   'nbdev.frontmatter.FrontmatterProc.begin': 'https://nbdev.fast.ai/frontmatter.html#frontmatterproc.begin',
+                                   'nbdev.frontmatter.FrontmatterProc.cell': 'https://nbdev.fast.ai/frontmatter.html#frontmatterproc.cell',
+                                   'nbdev.frontmatter.FrontmatterProc.end': 'https://nbdev.fast.ai/frontmatter.html#frontmatterproc.end'},
             'nbdev.imports': {},
             'nbdev.maker': { 'nbdev.maker.ModuleMaker': 'https://nbdev.fast.ai/maker.html#modulemaker',
                              'nbdev.maker.ModuleMaker._last_future': 'https://nbdev.fast.ai/maker.html#modulemaker._last_future',
@@ -157,8 +161,6 @@
                                   'nbdev.processors.filter_stream_': 'https://nbdev.fast.ai/processors.html#filter_stream_',
                                   'nbdev.processors.hide_': 'https://nbdev.fast.ai/processors.html#hide_',
                                   'nbdev.processors.hide_line': 'https://nbdev.fast.ai/processors.html#hide_line',
-                                  'nbdev.processors.infer_frontmatter': 'https://nbdev.fast.ai/processors.html#infer_frontmatter',
-                                  'nbdev.processors.infer_frontmatter.begin': 'https://nbdev.fast.ai/processors.html#infer_frontmatter.begin',
                                   'nbdev.processors.insert_frontmatter': 'https://nbdev.fast.ai/processors.html#insert_frontmatter',
                                   'nbdev.processors.insert_warning': 'https://nbdev.fast.ai/processors.html#insert_warning',
                                   'nbdev.processors.insert_warning.begin': 'https://nbdev.fast.ai/processors.html#insert_warning.begin',

nbdev/cli.py

@@ -11,6 +11,7 @@
 from .test import *
 from .clean import *
 from .quarto import refresh_quarto_yml
+from .frontmatter import FrontmatterProc
 
 from execnb.nbio import *
 from fastcore.meta import *
@@ -40,7 +41,7 @@ class FilterDefaults:
     def xtra_procs(self): return []
 
     def base_procs(self):
-        return [populate_language, infer_frontmatter, add_show_docs, insert_warning,
+        return [FrontmatterProc, populate_language, add_show_docs, insert_warning,
                 strip_ansi, hide_line, filter_stream_, rm_header_dash,
                 clean_show_doc, exec_show_docs, rm_export, clean_magics, hide_, add_links, strip_hidden_metadata]
 

nbdev/frontmatter.py

@@ -0,0 +1,62 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/09a_frontmatter.ipynb.
+
+# %% auto 0
+__all__ = ['FrontmatterProc']
+
+# %% ../nbs/09a_frontmatter.ipynb 2
+from .imports import *
+from .process import *
+
+from execnb.nbio import *
+from fastcore.imports import *
+import yaml
+
+# %% ../nbs/09a_frontmatter.ipynb 5
+_re_fm = re.compile(r'''^---\s*
+(.*\S+.*)
+---\s*$''', flags=re.DOTALL)
+
+def _fm2dict(s:str):
+    "Load YAML frontmatter into a `dict`"
+    match = _re_fm.search(s.strip())
+    return yaml.safe_load(match.group(1)) if match else {}
+
+def _md2dict(s:str):
+    "Convert H1 formatted markdown cell to frontmatter dict"
+    if '#' not in s: return {}
+    m = re.search(r'^#\s+(\S.*?)\s*$', s, flags=re.MULTILINE)
+    if not m: return {}
+    res = {'title': m.group(1)}
+    m = re.search(r'^>\s+(\S.*?)\s*$', s, flags=re.MULTILINE)
+    if m: res['description'] = m.group(1)
+    r = re.findall(r'^-\s+(\S.*?)\s*$', s, flags=re.MULTILINE)
+    if r:
+        try: res.update(yaml.safe_load('\n'.join(r)))
+        except Exception as e: warn(f'Failed to create YAML dict for:\n{r}\n\n{e}\n')
+    return res
+
+# %% ../nbs/09a_frontmatter.ipynb 6
+class FrontmatterProc(Processor):
+    "A YAML and formatted-markdown frontmatter processor"
+    def begin(self): self.fm = {}
+    def _default_exp_(self, cell, exp): self.default_exp = exp
+
+    def _update(self, f, cell):
+        s = cell.get('source')
+        if not s: return
+        d = f(cell.source)
+        if not d: return
+        self.fm.update(d)
+        cell.source = None
+
+    def cell(self, cell):
+        if cell.cell_type=='raw': self._update(_fm2dict, cell)
+        elif cell.cell_type=='markdown' and 'title' not in self.fm: self._update(_md2dict, cell)
+
+    def end(self):
+        self.nb.frontmatter_ = self.fm
+        if not self.fm: return
+        exp = getattr(self, 'default_exp', None)
+        if exp: self.fm.update({'output-file': exp+'.html'})
+        s = f'---\n{yaml.dump(self.fm)}\n---'
+        self.nb.cells.insert(0, mk_cell(s, 'raw'))

nbdev/processors.py

@@ -1,12 +1,12 @@
-# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/09_processors.ipynb.
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/09b_processors.ipynb.
 
 # %% auto 0
 __all__ = ['is_frontmatter', 'yml2dict', 'populate_language', 'insert_warning', 'cell_lang', 'add_show_docs', 'yaml_str',
-           'nb_fmdict', 'filter_fm', 'construct_fm', 'insert_frontmatter', 'infer_frontmatter', 'add_links',
-           'strip_ansi', 'strip_hidden_metadata', 'hide_', 'hide_line', 'filter_stream_', 'clean_magics',
-           'rm_header_dash', 'rm_export', 'clean_show_doc', 'exec_show_docs']
+           'nb_fmdict', 'filter_fm', 'construct_fm', 'insert_frontmatter', 'add_links', 'strip_ansi',
+           'strip_hidden_metadata', 'hide_', 'hide_line', 'filter_stream_', 'clean_magics', 'rm_header_dash',
+           'rm_export', 'clean_show_doc', 'exec_show_docs']
 
-# %% ../nbs/09_processors.ipynb 2
+# %% ../nbs/09b_processors.ipynb 2
 import ast
 
 from .config import *
@@ -21,7 +21,7 @@
 from fastcore.xtras import *
 import sys,yaml
 
-# %% ../nbs/09_processors.ipynb 7
+# %% ../nbs/09b_processors.ipynb 7
 _re_fm = re.compile(r'^---(.*\S+.*)---', flags=re.DOTALL)
 
 def is_frontmatter(nb):
@@ -39,7 +39,7 @@ def _get_frontmatter(nb):
     cell = first(is_frontmatter(nb))
     return cell,(yml2dict(cell.source) if cell else {})
 
-# %% ../nbs/09_processors.ipynb 8
+# %% ../nbs/09b_processors.ipynb 8
 _langs = 'bash|html|javascript|js|latex|markdown|perl|ruby|sh|svg'
 _lang_pattern = re.compile(rf'^\s*%%\s*({_langs})\s*$', flags=re.MULTILINE)
 
@@ -52,13 +52,13 @@ def cell(self, cell):
         if lang: cell.metadata.language = lang[0]
         else: cell.metadata.language = self.language
 
-# %% ../nbs/09_processors.ipynb 10
+# %% ../nbs/09b_processors.ipynb 10
 class insert_warning(Processor):
     "Insert Autogenerated Warning Into Notebook after the first cell."
     content = "<!-- WARNING: THIS FILE WAS AUTOGENERATED! DO NOT EDIT! -->"
     def begin(self): self.nb.cells.insert(1, mk_cell(self.content, 'markdown'))
 
-# %% ../nbs/09_processors.ipynb 14
+# %% ../nbs/09b_processors.ipynb 14
 _def_types = (ast.FunctionDef,ast.AsyncFunctionDef,ast.ClassDef)
 def _def_names(cell, shown):
     return [showdoc_nm(o) for o in concat(cell.parsed_())
@@ -70,7 +70,7 @@ def _get_nm(tree):
     else: val = try_attrs(i.value, 'id', 'func', 'attr')
     return f'{val}.{i.attr}' if isinstance(i, ast.Attribute) else i.id
 
-# %% ../nbs/09_processors.ipynb 15
+# %% ../nbs/09b_processors.ipynb 15
 def _show_docs(trees):
     return [t for t in trees if isinstance(t,ast.Expr) and nested_attr(t, 'value.func.id')=='show_doc']
 
@@ -90,22 +90,22 @@ def begin(self):
             for nm in _def_names(cell, shown_docs): nb.cells.insert(cell.idx_+1, mk_cell(f'show_doc({nm})'))
         nb.has_docs_ = shown_docs or exports
 
-# %% ../nbs/09_processors.ipynb 19
+# %% ../nbs/09b_processors.ipynb 19
 def yaml_str(s:str):
     "Create a valid YAML string from `s`"
     if s[0]=='"' and s[-1]=='"': return s
     res = s.replace('\\', '\\\\').replace('"', r'\"')
     return f'"{res}"'
 
-# %% ../nbs/09_processors.ipynb 20
+# %% ../nbs/09b_processors.ipynb 20
 _re_title = re.compile(r'^#\s+(.*)[\n\r]+(?:^>\s+(.*))?', flags=re.MULTILINE)
 
 def _celltyp(nb, cell_type): return L(nb.cells).filter(lambda c: c.cell_type == cell_type)
 def _istitle(cell): 
     txt = cell.get('source', '')
     return bool(_re_title.search(txt)) if txt else False
 
-# %% ../nbs/09_processors.ipynb 21
+# %% ../nbs/09b_processors.ipynb 21
 def nb_fmdict(nb, remove=True): 
     "Infer the front matter from a notebook's markdown formatting"
     md_cells = _celltyp(nb, 'markdown').filter(_istitle)
@@ -122,7 +122,7 @@ def nb_fmdict(nb, remove=True):
         return yml2dict('\n'.join([f"{k}: {flags[k]}" for k in flags]))
     else: return {}
 
-# %% ../nbs/09_processors.ipynb 24
+# %% ../nbs/09b_processors.ipynb 24
 def _replace_fm(d:dict, # dictionary you wish to conditionally change
                 k:str,  # key to check 
                 val:str,# value to check if d[k] == v
@@ -140,33 +140,33 @@ def _fp_alias(d):
     d = _replace_fm(d, 'hide', 'true', {'draft': 'true'})
     return d
 
-# %% ../nbs/09_processors.ipynb 26
+# %% ../nbs/09b_processors.ipynb 26
 def _fp_image(d):
     "Correct path of fastpages images"
     prefix = 'images/copied_from_nb/'
     if d.get('image', '').startswith(prefix): d['image'] = d['image'].replace(prefix, '')
     return d
 
-# %% ../nbs/09_processors.ipynb 28
+# %% ../nbs/09b_processors.ipynb 28
 def filter_fm(fmdict:dict):
     "Filter front matter"
     keys = ['title', 'description', 'author', 'image', 'categories', 'output-file', 'aliases', 'search', 'draft', 'comments']
     if not fmdict: return {}
     return filter_keys(fmdict, in_(keys))
 
-# %% ../nbs/09_processors.ipynb 29
+# %% ../nbs/09b_processors.ipynb 29
 def construct_fm(fmdict:dict):
     "Construct front matter from a dictionary"
     if not fmdict: return None
     return '---\n'+yaml.dump(fmdict)+'\n---'    
 
-# %% ../nbs/09_processors.ipynb 31
+# %% ../nbs/09b_processors.ipynb 31
 def insert_frontmatter(nb, fm_dict:dict):
     "Add frontmatter into notebook based on `filter_keys` that exist in `fmdict`."
     fm = construct_fm(fm_dict)
     if fm: nb.cells.insert(0, NbCell(0, dict(cell_type='raw', metadata={}, source=fm, directives_={})))
 
-# %% ../nbs/09_processors.ipynb 33
+# %% ../nbs/09b_processors.ipynb 33
 _re_defaultexp = re.compile(r'^\s*#\|\s*default_exp\s+(\S+)', flags=re.MULTILINE)
 
 def _default_exp(nb):
@@ -175,22 +175,7 @@ def _default_exp(nb):
     default_exp = first(code_src.filter().map(_re_defaultexp.search).filter())
     return default_exp.group(1) if default_exp else None
 
-# %% ../nbs/09_processors.ipynb 35
-_fp_convert = compose(_fp_alias, _fp_image)
-
-class infer_frontmatter(Processor):
-    "Insert front matter if it doesn't exist automatically from nbdev styled markdown."
-    def begin(self):
-        nb = self.nb
-        raw_fm_cell,raw_fm = _get_frontmatter(nb)
-        _exp = _default_exp(nb)
-        _fmdict = merge(_fp_convert(filter_fm(nb_fmdict(nb))), {'output-file': _exp+'.html'} if _exp else {}, raw_fm)
-        if not _fmdict: return
-        if raw_fm: raw_fm_cell['source'] = None
-        insert_frontmatter(nb, fm_dict=_fmdict)
-        nb.frontmatter_=_fmdict
-
-# %% ../nbs/09_processors.ipynb 44
+# %% ../nbs/09b_processors.ipynb 35
 def add_links(cell):
     "Add links to markdown cells"
     nl = NbdevLookup()
@@ -199,25 +184,25 @@ def add_links(cell):
         if hasattr(o, 'data') and hasattr(o['data'], 'text/markdown'):
             o.data['text/markdown'] = [nl.link_line(s) for s in o.data['text/markdown']]
 
-# %% ../nbs/09_processors.ipynb 47
+# %% ../nbs/09b_processors.ipynb 38
 _re_ansi_escape = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])')
 
 def strip_ansi(cell):
     "Strip Ansi Characters."
     for outp in cell.get('outputs', []):
         if outp.get('name')=='stdout': outp['text'] = [_re_ansi_escape.sub('', o) for o in outp.text]
 
-# %% ../nbs/09_processors.ipynb 49
+# %% ../nbs/09b_processors.ipynb 40
 def strip_hidden_metadata(cell):
     '''Strips "hidden" metadata property from code cells so it doesn't interfere with docs rendering'''
     if cell.cell_type == 'code' and 'metadata' in cell: cell.metadata.pop('hidden',None)
 
-# %% ../nbs/09_processors.ipynb 50
+# %% ../nbs/09b_processors.ipynb 41
 def hide_(cell):
     "Hide cell from output"
     del(cell['source'])
 
-# %% ../nbs/09_processors.ipynb 52
+# %% ../nbs/09b_processors.ipynb 43
 def _re_hideline(lang=None): return re.compile(fr'{langs[lang]}\|\s*hide_line\s*$', re.MULTILINE)
 
 def hide_line(cell):
@@ -226,22 +211,22 @@ def hide_line(cell):
     if cell.cell_type == 'code' and _re_hideline(lang).search(cell.source):
         cell.source = '\n'.join([c for c in cell.source.splitlines() if not _re_hideline(lang).search(c)])
 
-# %% ../nbs/09_processors.ipynb 55
+# %% ../nbs/09b_processors.ipynb 46
 def filter_stream_(cell, *words):
     "Remove output lines containing any of `words` in `cell` stream output"
     if not words: return
     for outp in cell.get('outputs', []):
         if outp.output_type == 'stream':
             outp['text'] = [l for l in outp.text if not re.search('|'.join(words), l)]
 
-# %% ../nbs/09_processors.ipynb 57
+# %% ../nbs/09b_processors.ipynb 48
 _magics_pattern = re.compile(r'^\s*(%%|%).*', re.MULTILINE)
 
 def clean_magics(cell):
     "A preprocessor to remove cell magic commands"
     if cell.cell_type == 'code': cell.source = _magics_pattern.sub('', cell.source).strip()
 
-# %% ../nbs/09_processors.ipynb 59
+# %% ../nbs/09b_processors.ipynb 50
 _re_hdr_dash = re.compile(r'^#+\s+.*\s+-\s*$', re.MULTILINE)
 
 def rm_header_dash(cell):
@@ -250,14 +235,14 @@ def rm_header_dash(cell):
         src = cell.source.strip()
         if cell.cell_type == 'markdown' and src.startswith('#') and src.endswith(' -'): del(cell['source'])
 
-# %% ../nbs/09_processors.ipynb 61
+# %% ../nbs/09b_processors.ipynb 52
 _hide_dirs = {'export','exporti', 'hide','default_exp'}
 
 def rm_export(cell):
     "Remove cells that are exported or hidden"
     if cell.directives_ and (cell.directives_.keys() & _hide_dirs): del(cell['source'])
 
-# %% ../nbs/09_processors.ipynb 63
+# %% ../nbs/09b_processors.ipynb 54
 _re_showdoc = re.compile(r'^show_doc', re.MULTILINE)
 def _is_showdoc(cell): return cell['cell_type'] == 'code' and _re_showdoc.search(cell.source)
 
@@ -266,7 +251,7 @@ def clean_show_doc(cell):
     if not _is_showdoc(cell): return
     cell.source = '#|output: asis\n#| echo: false\n' + cell.source
 
-# %% ../nbs/09_processors.ipynb 64
+# %% ../nbs/09b_processors.ipynb 55
 def _ast_contains(trees, types):
     for tree in trees:
         for node in ast.walk(tree):
@@ -287,7 +272,7 @@ def _do_eval(cell):
         return True
     if _show_docs(trees): return True
 
-# %% ../nbs/09_processors.ipynb 65
+# %% ../nbs/09b_processors.ipynb 56
 class exec_show_docs:
     "Execute cells needed for `show_docs` output, including exported cells and imports"
     def __init__(self, nb):

nbdev/test.py

@@ -14,7 +14,7 @@
 from .config import *
 from .doclinks import *
 from .process import NBProcessor, nb_lang
-from .processors import infer_frontmatter
+from .frontmatter import FrontmatterProc
 from logging import warning
 
 from execnb.nbio import *
@@ -31,7 +31,7 @@ def test_nb(fn,  # file name of notebook to test
     if basepath: sys.path.insert(0, str(basepath))
     if not IN_NOTEBOOK: os.environ["IN_TEST"] = '1'
     flags=set(L(skip_flags)) - set(L(force_flags))
-    nb = NBProcessor(fn, preprocs=infer_frontmatter, process=True).nb
+    nb = NBProcessor(fn, preprocs=FrontmatterProc, process=True).nb
     fm = getattr(nb, 'frontmatter_', {})
     if str2bool(fm.get('skip_exec', False)) or nb_lang(nb) != 'python': return True, 0
 

nbs/04b_doclinks.ipynb

@@ -14,7 +14,7 @@
    "metadata": {},
    "source": [
     "# doclinks\n",
-    "- Generating a documentation index from a module"
+    "> Generating a documentation index from a module"
    ]
   },
   {