moved sorting call outside of nbglob

Author: omlnaut

nbdev/doclinks.py

@@ -100,18 +100,16 @@ def _build_modidx(dest=None, nbs_path=None, skip_exists=False):
 
 # %% ../nbs/api/doclinks.ipynb 20
 @delegates(globtastic)
-def nbglob(path=None, skip_folder_re = '^[_.]', file_glob='*.ipynb', skip_file_re='^[_.]', key='nbs_path', as_path=False, sort_by=None, **kwargs):
+def nbglob(path=None, skip_folder_re = '^[_.]', file_glob='*.ipynb', skip_file_re='^[_.]', key='nbs_path', as_path=False, **kwargs):
     "Find all files in a directory matching an extension given a config key."
     path = Path(path or get_config()[key])
     recursive=get_config().recursive
     res = globtastic(path, file_glob=file_glob, skip_folder_re=skip_folder_re,
                      skip_file_re=skip_file_re, recursive=recursive, **kwargs)
     res = res.map(Path) if as_path else res
-    if sort_by is not None:
-        res.sort(key=sort_by)
     return res
 
-# %% ../nbs/api/doclinks.ipynb 23
+# %% ../nbs/api/doclinks.ipynb 22
 def nbglob_cli(
     path:str=None, # Path to notebooks
     symlinks:bool=False, # Follow symlinks?
@@ -125,24 +123,24 @@ def nbglob_cli(
     return nbglob(path, symlinks=symlinks, file_glob=file_glob, file_re=file_re, folder_re=folder_re,
                   skip_file_glob=skip_file_glob, skip_file_re=skip_file_re, skip_folder_re=skip_folder_re)
 
-# %% ../nbs/api/doclinks.ipynb 24
+# %% ../nbs/api/doclinks.ipynb 23
 @call_parse
 @delegates(nbglob_cli)
 def nbdev_export(
     path:str=None, # Path or filename
     **kwargs):
     "Export notebooks in `path` to Python modules"
     if os.environ.get('IN_TEST',0): return
-    files = nbglob(path=path, sort_by=lambda path_str: Path(path_str).name, **kwargs)
+    files = nbglob(path=path, **kwargs)#.sorted(lambda path_str: Path(path_str).name)
     for f in files: nb_export(f)
     add_init(get_config().lib_path)
     _build_modidx()
 
-# %% ../nbs/api/doclinks.ipynb 26
+# %% ../nbs/api/doclinks.ipynb 27
 import importlib,ast
 from functools import lru_cache
 
-# %% ../nbs/api/doclinks.ipynb 27
+# %% ../nbs/api/doclinks.ipynb 28
 def _find_mod(mod):
     mp,_,mr = mod.partition('/')
     spec = importlib.util.find_spec(mp)
@@ -165,7 +163,7 @@ def _get_exps(mod):
 
 def _lineno(sym, fname): return _get_exps(fname).get(sym, None) if fname else None
 
-# %% ../nbs/api/doclinks.ipynb 29
+# %% ../nbs/api/doclinks.ipynb 30
 def _qual_sym(s, settings):
     if not isinstance(s,tuple): return s
     nb,py = s
@@ -180,10 +178,10 @@ def _qual_syms(entries):
     if 'doc_host' not in settings: return entries
     return {'syms': {mod:_qual_mod(d, settings) for mod,d in entries['syms'].items()}, 'settings':settings}
 
-# %% ../nbs/api/doclinks.ipynb 30
+# %% ../nbs/api/doclinks.ipynb 31
 _re_backticks = re.compile(r'`([^`\s]+)`')
 
-# %% ../nbs/api/doclinks.ipynb 31
+# %% ../nbs/api/doclinks.ipynb 32
 @lru_cache(None)
 class NbdevLookup:
     "Mapping from symbol names to docs and source URLs"

nbs/api/doclinks.ipynb

@@ -306,41 +306,16 @@
    "source": [
     "#|export\n",
     "@delegates(globtastic)\n",
-    "def nbglob(path=None, skip_folder_re = '^[_.]', file_glob='*.ipynb', skip_file_re='^[_.]', key='nbs_path', as_path=False, sort_by=None, **kwargs):\n",
+    "def nbglob(path=None, skip_folder_re = '^[_.]', file_glob='*.ipynb', skip_file_re='^[_.]', key='nbs_path', as_path=False, **kwargs):\n",
     "    \"Find all files in a directory matching an extension given a config key.\"\n",
     "    path = Path(path or get_config()[key])\n",
     "    recursive=get_config().recursive\n",
     "    res = globtastic(path, file_glob=file_glob, skip_folder_re=skip_folder_re,\n",
     "                     skip_file_re=skip_file_re, recursive=recursive, **kwargs)\n",
     "    res = res.map(Path) if as_path else res\n",
-    "    if sort_by is not None:\n",
-    "        res.sort(key=sort_by)\n",
     "    return res"
    ]
   },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "`globtastic` uses `glob.glob`, which in turn uses `os.listdir` under the hood. As the [documentation](https://docs.python.org/3/library/os.html#os.listdir) states, the order of results returned by `os.listdir` is arbitrary, hence we need to be able to sort the notebook-paths when needed:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import copy\n",
-    "\n",
-    "globtastic_save = copy.copy(globtastic)\n",
-    "\n",
-    "globtastic = lambda *args, **kwargs: ['../../8.ipynb', 'api/3.ipynb', '4.ipynb']\n",
-    "assert nbglob(sort_by=lambda path_str: Path(path_str).name) == ['api/3.ipynb', '4.ipynb', '../../8.ipynb']\n",
-    "\n",
-    "globtastic = globtastic_save"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -376,12 +351,57 @@
     "    **kwargs):\n",
     "    \"Export notebooks in `path` to Python modules\"\n",
     "    if os.environ.get('IN_TEST',0): return\n",
-    "    files = nbglob(path=path, sort_by=lambda path_str: Path(path_str).name, **kwargs)\n",
+    "    files = sorted(nbglob(path=path, **kwargs), key=lambda path_str: Path(path_str).name)\n",
     "    for f in files: nb_export(f)\n",
     "    add_init(get_config().lib_path)\n",
     "    _build_modidx()"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "When exporting the notebooks, they should be exported in order of the filename to ensure deterministic build behaviour. The below test patches `nbglob` to return a non-ordered list, then checks that `nb_export` is still called on the notebooks in the right order:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from copy import copy\n",
+    "\n",
+    "\n",
+    "class EscapeException(Exception):\n",
+    "    pass\n",
+    "\n",
+    "def nb_export_mock(nb_path: str):\n",
+    "    assert nb_path==expected_order.pop()\n",
+    "    \n",
+    "    # escape from the outer function as the tests are done\n",
+    "    if not expected_order:\n",
+    "        raise EscapeException()\n",
+    "        \n",
+    "\n",
+    "nbglob_save = copy(nbglob)\n",
+    "nb_export_save = copy(nb_export)\n",
+    "\n",
+    "nbglob = lambda *args, **kwargs: ['../../8.ipynb', 'api/3.ipynb', '4.ipynb']\n",
+    "\n",
+    "expected_order = ['api/3.ipynb', '4.ipynb', '../../8.ipynb']\n",
+    "expected_order.reverse()\n",
+    "nb_export = nb_export_mock\n",
+    "\n",
+    "try:\n",
+    "    nbdev_export()\n",
+    "except EscapeException:\n",
+    "    pass # this silences the exception used to end nbdev_export early\n",
+    "\n",
+    "nbglob = nbglob_save\n",
+    "nb_export = nb_export_save"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -578,7 +598,7 @@
       "text/markdown": [
        "---\n",
        "\n",
-       "[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L209){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
+       "[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L211){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
        "\n",
        "### NbdevLookup.doc\n",
        "\n",
@@ -589,7 +609,7 @@
       "text/plain": [
        "---\n",
        "\n",
-       "[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L209){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
+       "[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L211){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
        "\n",
        "### NbdevLookup.doc\n",
        "\n",
@@ -672,7 +692,7 @@
       "text/markdown": [
        "---\n",
        "\n",
-       "[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L214){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
+       "[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L216){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
        "\n",
        "### NbdevLookup.code\n",
        "\n",
@@ -683,7 +703,7 @@
       "text/plain": [
        "---\n",
        "\n",
-       "[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L214){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
+       "[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L216){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
        "\n",
        "### NbdevLookup.code\n",
        "\n",
@@ -731,7 +751,7 @@
       "text/markdown": [
        "---\n",
        "\n",
-       "[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L231){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
+       "[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L233){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
        "\n",
        "### NbdevLookup.linkify\n",
        "\n",
@@ -740,7 +760,7 @@
       "text/plain": [
        "---\n",
        "\n",
-       "[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L231){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
+       "[source](https://github.com/fastai/nbdev/blob/master/nbdev/doclinks.py#L233){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
        "\n",
        "### NbdevLookup.linkify\n",
        "\n",