Merge branch 'fastai:master' into master

Author: omlnaut

CHANGELOG.md

@@ -2,6 +2,51 @@
 
 <!-- do not remove -->
 
+## 2.3.9
+
+### New Features
+
+- utility that creates a `requirements.txt` file from `settings.ini` ([#1202](https://github.com/fastai/nbdev/pull/1202)), thanks to [@hamelsmu](https://github.com/hamelsmu)
+- user-friendly error if py file has `# %%` comments with unexpected format ([#1211](https://github.com/fastai/nbdev/pull/1211)), thanks to [@seeM](https://github.com/seeM)
+- add parameter for name to `nb_export` ([#1204](https://github.com/fastai/nbdev/pull/1204)), thanks to [@hamelsmu](https://github.com/hamelsmu)
+- ensure newline at end of `_modidx.py` ([#1186](https://github.com/fastai/nbdev/issues/1186))
+
+### Bugs Squashed
+
+- end `sidebar.yml` with newline ([#1212](https://github.com/fastai/nbdev/pull/1212)), thanks to [@seeM](https://github.com/seeM)
+- fix: incorrect regex pattern for setting `output-file` ([#1210](https://github.com/fastai/nbdev/pull/1210)), thanks to [@seeM](https://github.com/seeM)
+- ensure newline at end of `_modidx.py` ([#1209](https://github.com/fastai/nbdev/pull/1209)), thanks to [@seeM](https://github.com/seeM)
+- fix: `nbdev_install_quarto` may install and remove unrelated packages ([#1208](https://github.com/fastai/nbdev/pull/1208)), thanks to [@seeM](https://github.com/seeM)
+- fix: key error if widgets is missing `state` ([#1207](https://github.com/fastai/nbdev/pull/1207)), thanks to [@seeM](https://github.com/seeM)
+-`nbdev_install_quarto` may install and remove unrelated packages ([#1182](https://github.com/fastai/nbdev/issues/1182))
+- Key error if widgets is missing `state` ([#1167](https://github.com/fastai/nbdev/issues/1167))
+
+
+## 2.3.8
+
+### New Features
+
+- better error messages for `nbdev_migrate` ([#1177](https://github.com/fastai/nbdev/issues/1177))
+- experimental: Users can provide extra processors via the `procs` key in `settings.ini` ([#1157](https://github.com/fastai/nbdev/pull/1157)), thanks to [@seeM](https://github.com/seeM)
+- enable Documentation Only Sites With `nbdev` ( + tutorial ) ([#1121](https://github.com/fastai/nbdev/pull/1121)), thanks to [@hamelsmu](https://github.com/hamelsmu)
+- support non-library projects ([#1119](https://github.com/fastai/nbdev/issues/1119))
+- throw a warning when imports and code are mixed in a cell ([#714](https://github.com/fastai/nbdev/issues/714))
+
+### Bugs Squashed
+
+- fix duplicated sections in the sidebar ([#1165](https://github.com/fastai/nbdev/pull/1165)), thanks to [@seeM](https://github.com/seeM)
+- setting `#| echo` in a cell with `show_doc` causes a Quarto error ([#1163](https://github.com/fastai/nbdev/issues/1163))
+- fix copying of index assets ([#1143](https://github.com/fastai/nbdev/pull/1143)), thanks to [@hamelsmu](https://github.com/hamelsmu)
+- images in index.ipynb causing deployments issues ([#1140](https://github.com/fastai/nbdev/issues/1140))
+- clean takes forever on notebooks with large output ([#1132](https://github.com/fastai/nbdev/issues/1132))
+- `nbdev_update` includes folders starting with `_` or `.` (e.g. `.ipynb_checkpoints`) ([#1130](https://github.com/fastai/nbdev/pull/1130)), thanks to [@seeM](https://github.com/seeM)
+- `nbdev_new` defaults bool parameters to `False` (e.g. `put_version_in_init`) ([#1129](https://github.com/fastai/nbdev/pull/1129)), thanks to [@seeM](https://github.com/seeM)
+- `black_formatting` setting is ignored ([#1122](https://github.com/fastai/nbdev/pull/1122)), thanks to [@jmoralez](https://github.com/jmoralez)
+- `nbdev_readme()` fails on the second run for the notebook with support files (e.g. Fig image). ([#1106](https://github.com/fastai/nbdev/issues/1106))
+- `nbdev_new` fails with `AttributeError: path_` ([#1063](https://github.com/fastai/nbdev/issues/1063))
+- fix #1041 ([#1049](https://github.com/fastai/nbdev/pull/1049)), thanks to [@seeM](https://github.com/seeM)
+
+
 ## 2.3.7
 
 ### New Features

nbdev/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "2.3.8"
+__version__ = "2.3.10"
 
 from .doclinks import nbdev_export
 from .showdoc import show_doc

nbdev/_modidx.py

@@ -277,7 +277,8 @@
                                'nbdev.release.release_gh': ('api/release.html#release_gh', 'nbdev/release.py'),
                                'nbdev.release.release_git': ('api/release.html#release_git', 'nbdev/release.py'),
                                'nbdev.release.release_pypi': ('api/release.html#release_pypi', 'nbdev/release.py'),
-                               'nbdev.release.write_conda_meta': ('api/release.html#write_conda_meta', 'nbdev/release.py')},
+                               'nbdev.release.write_conda_meta': ('api/release.html#write_conda_meta', 'nbdev/release.py'),
+                               'nbdev.release.write_requirements': ('api/release.html#write_requirements', 'nbdev/release.py')},
             'nbdev.serve': { 'nbdev.serve._is_qpy': ('api/serve.html#_is_qpy', 'nbdev/serve.py'),
                              'nbdev.serve._proc_file': ('api/serve.html#_proc_file', 'nbdev/serve.py'),
                              'nbdev.serve.proc_nbs': ('api/serve.html#proc_nbs', 'nbdev/serve.py')},
@@ -331,4 +332,4 @@
                             'nbdev.sync.nbdev_update': ('api/sync.html#nbdev_update', 'nbdev/sync.py')},
             'nbdev.test': { 'nbdev.test._keep_file': ('api/test.html#_keep_file', 'nbdev/test.py'),
                             'nbdev.test.nbdev_test': ('api/test.html#nbdev_test', 'nbdev/test.py'),
-                            'nbdev.test.test_nb': ('api/test.html#test_nb', 'nbdev/test.py')}}}
+                            'nbdev.test.test_nb': ('api/test.html#test_nb', 'nbdev/test.py')}}}

nbdev/doclinks.py

@@ -49,13 +49,15 @@ def _iter_py_cells(p):
     cells = p.read_text().split("\n# %% ")
     for cell in cells[1:]:
         top,code = cell.split('\n', 1)
-        nb,idx = top.split()
+        try: nb,idx = top.split()
+        except ValueError: raise ValueError(f"Unexpected format in '{p}' at cell:\n```\n# %% {cell.strip()}.\n```\n"
+                                            "The expected format is: '# %% {nb_path} {cell_idx}'.")
         nb_path = None if nb=='auto' else (p.parent/nb).resolve()  # NB paths are stored relative to .py file
         if code.endswith('\n'): code=code[:-1]
         yield AttrDict(nb=nb, idx=int(idx), code=code, nb_path=nb_path, py_path=p.resolve())
 
 # %% ../nbs/api/doclinks.ipynb 11
-def _nbpath2html(p): return p.with_name(re.sub(r'\d+[a-zA-Z0-9]*_', '', p.name.lower())).with_suffix('.html')
+def _nbpath2html(p): return p.with_name(re.sub(r'^\d+[a-zA-Z0-9]*_', '', p.name.lower())).with_suffix('.html')
 
 # %% ../nbs/api/doclinks.ipynb 13
 def _get_modidx(py_path, code_root, nbs_path):
@@ -96,7 +98,7 @@ def _build_modidx(dest=None, nbs_path=None, skip_exists=False):
     code_root = dest.parent.resolve()
     for file in globtastic(dest, file_glob="*.py", skip_file_re='^_', skip_folder_re="\.ipynb_checkpoints"):
         res['syms'].update(_get_modidx((dest.parent/file).resolve(), code_root, nbs_path=nbs_path))
-    idxfile.write_text("# Autogenerated by nbdev\n\nd = "+pformat(res, width=140, indent=2, compact=True))
+    idxfile.write_text("# Autogenerated by nbdev\n\nd = "+pformat(res, width=140, indent=2, compact=True)+'\n')
 
 # %% ../nbs/api/doclinks.ipynb 20
 @delegates(globtastic)

nbdev/export.py

@@ -41,19 +41,19 @@ def black_format(cell, # Cell to format
         except: pass
 
 # %% ../nbs/api/export.ipynb 9
-def nb_export(nbname, lib_path=None, procs=black_format, debug=False, mod_maker=ModuleMaker):
+def nb_export(nbname, lib_path=None, procs=black_format, debug=False, mod_maker=ModuleMaker, name=None):
     "Create module(s) from notebook"
     if lib_path is None: lib_path = get_config().lib_path
     exp = ExportModuleProc()
     nb = NBProcessor(nbname, [exp]+L(procs), debug=debug)
     nb.process()
     for mod,cells in exp.modules.items():
         all_cells = exp.in_all[mod]
-        name = getattr(exp, 'default_exp', None) if mod=='#' else mod
-        if not name:
+        nm = ifnone(name, getattr(exp, 'default_exp', None) if mod=='#' else mod)
+        if not nm:
             warn(f"Notebook '{nbname}' uses `#|export` without `#|default_exp` cell.\n"
                  "Note nbdev2 no longer supports nbdev1 syntax. Run `nbdev_migrate` to upgrade.\n"
                  "See https://nbdev.fast.ai/getting_started.html for more information.")
             return
-        mm = mod_maker(dest=lib_path, name=name, nb_path=nbname, is_new=mod=='#')
+        mm = mod_maker(dest=lib_path, name=nm, nb_path=nbname, is_new=bool(name) or mod=='#')
         mm.make(cells, all_cells, lib_path=lib_path)

nbdev/migrate.py

@@ -17,7 +17,7 @@
 # %% ../nbs/api/migrate.ipynb 5
 def _cat_slug(fmdict):
     "Get the partial slug from the category front matter."
-    slug = '/'.join(sorted(fmdict.get('categories', '')))
+    slug = '/'.join(fmdict.get('categories', ''))
     return '/' + slug if slug else '' 
 
 # %% ../nbs/api/migrate.ipynb 7

nbdev/processors.py

@@ -211,7 +211,7 @@ def end(self):
             mimetype = 'application/vnd.jupyter.widget-state+json'
             old = nested_idx(self.nb.metadata, 'widgets', mimetype) or {'state': {}}
             new = Widget.get_manager_state(drop_defaults=True)
-            widgets = {**old, **new, 'state': {**old['state'], **new['state']}}
+            widgets = {**old, **new, 'state': {**old.get('state', {}), **new['state']}}
             self.nb.metadata['widgets'] = {mimetype: widgets}
 
 # %% ../nbs/api/processors.ipynb 42

nbdev/quarto.py

@@ -30,11 +30,11 @@ def _sprun(cmd):
 
 def _install_linux():
     system(f'curl -LO {BASE_QUARTO_URL}quarto-linux-amd64.deb')
-    system('sudo dpkg -i *64.deb && rm *64.deb')
+    system('sudo dpkg -i quarto-linux-amd64.deb && rm quarto-linux-amd64.deb')
 
 def _install_mac():
     system(f'curl -LO {BASE_QUARTO_URL}quarto-macos.pkg')
-    system('sudo installer -pkg quarto-macos.pkg -target /')
+    system('sudo installer -pkg quarto-macos.pkg -target / && rm quarto-macos.pkg')
 
 @call_parse
 def install_quarto():
@@ -104,7 +104,7 @@ def _f(a,b): return Path(a),b
 
     yml_path = path/'sidebar.yml'
     yml = "website:\n  sidebar:\n    contents:\n"
-    yml += '\n'.join(f'      {o}' for o in res)
+    yml += '\n'.join(f'      {o}' for o in res)+'\n'
     if printit: return print(yml)
     yml_path.write_text(yml)
 

nbdev/release.py

@@ -2,8 +2,8 @@
 
 # %% auto 0
 __all__ = ['GH_HOST', 'Release', 'changelog', 'release_git', 'release_gh', 'pypi_json', 'latest_pypi', 'pypi_details',
-           'conda_output_path', 'write_conda_meta', 'anaconda_upload', 'release_conda', 'chk_conda_rel', 'release_pypi',
-           'release_both', 'bump_version', 'nbdev_bump_version']
+           'conda_output_path', 'write_conda_meta', 'write_requirements', 'anaconda_upload', 'release_conda',
+           'chk_conda_rel', 'release_pypi', 'release_both', 'bump_version', 'nbdev_bump_version']
 
 # %% ../nbs/api/release.ipynb 14
 from fastcore.all import *
@@ -178,9 +178,9 @@ def _run(cmd):
     return res
 
 # %% ../nbs/api/release.ipynb 38
-def conda_output_path(name):
+def conda_output_path(name, build='build'):
     "Output path for conda build"
-    return run(f'conda build --output {name}').strip().replace('\\', '/')
+    return run(f'conda {build} --output {name}').strip().replace('\\', '/')
 
 # %% ../nbs/api/release.ipynb 39
 def _write_yaml(path, name, d1, d2):
@@ -198,6 +198,9 @@ def _get_conda_meta():
     name,ver = cfg.lib_name,cfg.version
     url = cfg.doc_host or cfg.git_url
 
+    doc_url = (cfg.doc_host + cfg.doc_baseurl) if (cfg.doc_host and cfg.doc_baseurl) else url
+    dev_url = cfg.git_url if cfg.git_url else url
+
     reqs = ['pip', 'python', 'packaging']
     if cfg.get('requirements'): reqs += cfg.requirements.split()
     if cfg.get('conda_requirements'): reqs += cfg.conda_requirements.split()
@@ -222,7 +225,7 @@ def _get_conda_meta():
         'about': {
             'license': 'Apache Software',
             'license_family': 'APACHE',
-            'home': url, 'doc_url': url, 'dev_url': url,
+            'home': dev_url, 'doc_url': doc_url, 'dev_url': dev_url,
             'summary': cfg.get('description'),
             'description': descr
         },
@@ -236,6 +239,15 @@ def write_conda_meta(path='conda'):
     _write_yaml(path, *_get_conda_meta())
 
 # %% ../nbs/api/release.ipynb 43
+# This function is used as a utility for creating HF spaces.
+def write_requirements(directory=None):
+    "Writes a `requirements.txt` file to `directory` based on settings.ini."
+    cfg = get_config()
+    d = Path(directory) if directory else cfg.config_path
+    req = '\n'.join([cfg.get(k, '').replace(' ', '\n') for k in ['requirements', 'pip_requirements']])
+    (d/'requirements.txt').mk_write(req)
+
+# %% ../nbs/api/release.ipynb 45
 def anaconda_upload(name, loc=None, user=None, token=None, env_token=None):
     "Upload `name` to anaconda"
     user = f'-u {user} ' if user else ''
@@ -245,7 +257,7 @@ def anaconda_upload(name, loc=None, user=None, token=None, env_token=None):
     if not loc: raise Exception("Failed to find output")
     return _run(f'anaconda {token} upload {user} {loc} --skip-existing')
 
-# %% ../nbs/api/release.ipynb 44
+# %% ../nbs/api/release.ipynb 47
 @call_parse
 def release_conda(
     path:str='conda', # Path where package will be created
@@ -260,20 +272,22 @@ def release_conda(
     write_conda_meta(path)
     out = f"Done. Next steps:\n```\ncd {path}\n"""
     os.chdir(path)
-    loc = conda_output_path(name)
-    out_upl = f"anaconda upload {loc}"
     build = 'mambabuild' if mambabuild else 'build'
-    if not do_build: return print(f"{out}conda {build} .\n{out_upl}\n```")
-
-    print(f"conda {build} --no-anaconda-upload {build_args} {name}")
-    res = _run(f"conda {build} --no-anaconda-upload {build_args} {name}")
-    if skip_upload: return
+    if not do_build: return print(f"{out}conda {build} {name}")
+
+    cmd = f"conda {build} --output-folder out --no-anaconda-upload {build_args} {name}"
+    print(cmd)
+    res = _run(cmd)
+    outs = globtastic('out', file_glob='*.tar.bz2')
+    assert len(outs)==1
+    loc = outs[0]
+    if skip_upload: return print(loc)
     if not upload_user: upload_user = get_config().conda_user
     if not upload_user: return print("`conda_user` not in settings.ini and no `upload_user` passed. Cannot upload")
     if 'anaconda upload' not in res: return print(f"{res}\n\Failed. Check auto-upload not set in .condarc. Try `--do_build False`.")
     return anaconda_upload(name, loc)
 
-# %% ../nbs/api/release.ipynb 45
+# %% ../nbs/api/release.ipynb 48
 def chk_conda_rel(
     nm:str,  # Package name on pypi
     apkg:str=None,  # Anaconda Package (defaults to {nm})
@@ -287,7 +301,7 @@ def chk_conda_rel(
     pypitag = latest_pypi(nm)
     if force or not condatag or pypitag > max(condatag): return f'{pypitag}'
 
-# %% ../nbs/api/release.ipynb 47
+# %% ../nbs/api/release.ipynb 50
 @call_parse
 def release_pypi(
     repository:str="pypi" # Respository to upload to (defined in ~/.pypirc)
@@ -297,7 +311,7 @@ def release_pypi(
     system(f'cd {_dir}  && rm -rf dist && python setup.py sdist bdist_wheel')
     system(f'twine upload --repository {repository} {_dir}/dist/*')
 
-# %% ../nbs/api/release.ipynb 48
+# %% ../nbs/api/release.ipynb 51
 @call_parse
 def release_both(
     path:str='conda', # Path where package will be created
@@ -313,15 +327,15 @@ def release_both(
     release_conda.__wrapped__(path, do_build=do_build, build_args=build_args, skip_upload=skip_upload, mambabuild=mambabuild, upload_user=upload_user)
     nbdev_bump_version.__wrapped__()
 
-# %% ../nbs/api/release.ipynb 50
+# %% ../nbs/api/release.ipynb 53
 def bump_version(version, part=2, unbump=False):
     version = version.split('.')
     incr = -1 if unbump else 1
     version[part] = str(int(version[part]) + incr)
     for i in range(part+1, 3): version[i] = '0'
     return '.'.join(version)
 
-# %% ../nbs/api/release.ipynb 51
+# %% ../nbs/api/release.ipynb 54
 @call_parse
 def nbdev_bump_version(
     part:int=2,  # Part of version to bump

nbs/api/doclinks.ipynb

@@ -137,7 +137,9 @@
     "    cells = p.read_text().split(\"\\n# %% \")\n",
     "    for cell in cells[1:]:\n",
     "        top,code = cell.split('\\n', 1)\n",
-    "        nb,idx = top.split()\n",
+    "        try: nb,idx = top.split()\n",
+    "        except ValueError: raise ValueError(f\"Unexpected format in '{p}' at cell:\\n```\\n# %% {cell.strip()}.\\n```\\n\"\n",
+    "                                            \"The expected format is: '# %% {nb_path} {cell_idx}'.\")\n",
     "        nb_path = None if nb=='auto' else (p.parent/nb).resolve()  # NB paths are stored relative to .py file\n",
     "        if code.endswith('\\n'): code=code[:-1]\n",
     "        yield AttrDict(nb=nb, idx=int(idx), code=code, nb_path=nb_path, py_path=p.resolve())"
@@ -162,7 +164,7 @@
    "outputs": [],
    "source": [
     "#|export\n",
-    "def _nbpath2html(p): return p.with_name(re.sub(r'\\d+[a-zA-Z0-9]*_', '', p.name.lower())).with_suffix('.html')"
+    "def _nbpath2html(p): return p.with_name(re.sub(r'^\\d+[a-zA-Z0-9]*_', '', p.name.lower())).with_suffix('.html')"
    ]
   },
   {
@@ -237,7 +239,7 @@
     "    code_root = dest.parent.resolve()\n",
     "    for file in globtastic(dest, file_glob=\"*.py\", skip_file_re='^_', skip_folder_re=\"\\.ipynb_checkpoints\"):\n",
     "        res['syms'].update(_get_modidx((dest.parent/file).resolve(), code_root, nbs_path=nbs_path))\n",
-    "    idxfile.write_text(\"# Autogenerated by nbdev\\n\\nd = \"+pformat(res, width=140, indent=2, compact=True))"
+    "    idxfile.write_text(\"# Autogenerated by nbdev\\n\\nd = \"+pformat(res, width=140, indent=2, compact=True)+'\\n')"
    ]
   },
   {