Merge pull request #698 from seeM/jupyter-hooks

fixes #697

Author: jph00

nbdev/_modidx.py

@@ -12,6 +12,7 @@
                                    'nbdev_trust=nbdev.clean:nbdev_trust\n'
                                    'nbdev_clean=nbdev.clean:nbdev_clean\n'
                                    'nbdev_install_hooks=nbdev.clean:nbdev_install_hooks\n'
+                                   'nbdev_install_jupyter_hooks=nbdev.clean:nbdev_install_jupyter_hooks\n'
                                    'nbdev_filter=nbdev.cli:nbdev_filter\n'
                                    'nbdev_quarto=nbdev.cli:nbdev_quarto\n'
                                    'nbdev_ghp_deploy=nbdev.cli:nbdev_ghp_deploy\n'
@@ -56,9 +57,11 @@
                 'tst_flags': 'notest',
                 'user': 'fastai',
                 'version': '2.0.6'},
-  'syms': { 'nbdev.clean': { 'nbdev.clean.clean_nb': 'https://nbdev.fast.ai/clean.html#clean_nb',
+  'syms': { 'nbdev.clean': { 'nbdev.clean.clean_jupyter': 'https://nbdev.fast.ai/clean.html#clean_jupyter',
+                             'nbdev.clean.clean_nb': 'https://nbdev.fast.ai/clean.html#clean_nb',
                              'nbdev.clean.nbdev_clean': 'https://nbdev.fast.ai/clean.html#nbdev_clean',
                              'nbdev.clean.nbdev_install_hooks': 'https://nbdev.fast.ai/clean.html#nbdev_install_hooks',
+                             'nbdev.clean.nbdev_install_jupyter_hooks': 'https://nbdev.fast.ai/clean.html#nbdev_install_jupyter_hooks',
                              'nbdev.clean.nbdev_trust': 'https://nbdev.fast.ai/clean.html#nbdev_trust',
                              'nbdev.clean.process_write': 'https://nbdev.fast.ai/clean.html#process_write'},
             'nbdev.cli': { 'nbdev.cli.FilterDefaults': 'https://nbdev.fast.ai/cli.html#filterdefaults',

nbdev/clean.py

@@ -1,7 +1,8 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/11_clean.ipynb.
 
 # %% auto 0
-__all__ = ['nbdev_trust', 'clean_nb', 'process_write', 'nbdev_clean', 'nbdev_install_hooks']
+__all__ = ['nbdev_trust', 'clean_nb', 'process_write', 'nbdev_clean', 'nbdev_install_hooks', 'clean_jupyter',
+           'nbdev_install_jupyter_hooks']
 
 # %% ../nbs/11_clean.ipynb 2
 import warnings,stat
@@ -96,6 +97,13 @@ def process_write(warn_msg, proc_nb, f_in, f_out=None, disp=False):
         warn(e)
 
 # %% ../nbs/11_clean.ipynb 21
+def _nbdev_clean(nb, **kwargs):
+    allowed_metadata_keys = config_key("allowed_metadata_keys", '', missing_ok=True, path=False).split()
+    allowed_cell_metadata_keys = config_key("allowed_cell_metadata_keys", '', missing_ok=True, path=False).split()
+    return clean_nb(nb, allowed_metadata_keys=allowed_metadata_keys,
+                    allowed_cell_metadata_keys=allowed_cell_metadata_keys, **kwargs)
+
+# %% ../nbs/11_clean.ipynb 22
 @call_parse
 def nbdev_clean(
     fname:str=None, # A notebook name or glob to clean
@@ -105,18 +113,14 @@ def nbdev_clean(
 ):
     "Clean all notebooks in `fname` to avoid merge conflicts"
     # Git hooks will pass the notebooks in stdin
-    allowed_metadata_keys = config_key("allowed_metadata_keys", default='', missing_ok=True, path=False).split()
-    allowed_cell_metadata_keys = config_key("allowed_cell_metadata_keys", default='', missing_ok=True, path=False).split()
-    _clean = partial(clean_nb, clear_all=clear_all,
-                     allowed_metadata_keys=allowed_metadata_keys,
-                     allowed_cell_metadata_keys=allowed_cell_metadata_keys)
+    _clean = partial(_nbdev_clean, clear_all=clear_all)
     _write = partial(process_write, warn_msg='Failed to clean notebook', proc_nb=_clean)
     if stdin: return _write(f_in=sys.stdin, f_out=sys.stdout)
 
     if fname is None: fname = config_key("nbs_path", '.', missing_ok=True)
     for f in globtastic(fname, file_glob='*.ipynb', skip_folder_re='^[_.]'): _write(f_in=f, disp=disp)
 
-# %% ../nbs/11_clean.ipynb 23
+# %% ../nbs/11_clean.ipynb 24
 @call_parse
 def nbdev_install_hooks():
     "Install git hooks to clean and trust notebooks automatically"
@@ -149,3 +153,38 @@ def nbdev_install_hooks():
     run(cmd)
     print("Hooks are installed and repo's .gitconfig is now trusted")
     (nb_path/'.gitattributes').write_text("**/*.ipynb filter=clean-nbs\n**/*.ipynb diff=ipynb\n")
+
+# %% ../nbs/11_clean.ipynb 25
+def clean_jupyter(path, model, **kwargs):
+    "Clean Jupyter `model` pre save to `path`"
+    get_config.cache_clear() # Reset Jupyter's cache
+    try: cfg = get_config(path=path)
+    except FileNotFoundError: return
+    in_nbdev_repo = 'nbs_path' in cfg
+    jupyter_hooks = str2bool(cfg.get('jupyter_hooks', True))
+    is_nb_v4 = (model['type'],model['content']['nbformat']) == ('notebook',4)
+    if in_nbdev_repo and jupyter_hooks and is_nb_v4: _nbdev_clean(model['content'])
+
+# %% ../nbs/11_clean.ipynb 27
+def _nested_setdefault(o, attr, default):
+    "Same as `setdefault`, but if `attr` includes a `.`, then looks inside nested objects"
+    attrs = attr.split('.')
+    for a in attrs[:-1]: o = o.setdefault(a, type(o)())
+    return o.setdefault(attrs[-1], default)
+
+# %% ../nbs/11_clean.ipynb 31
+@call_parse
+def nbdev_install_jupyter_hooks():
+    "Install Jupyter hooks to clean notebooks on save"
+    cfg_path = Path.home()/'.jupyter'
+    cfg_fns = [cfg_path/f'jupyter_{o}_config.json' for o in ('notebook','server')]
+    attr,hook = 'ContentsManager.pre_save_hook','nbdev.clean.clean_jupyter'
+    for fn in cfg_fns:
+        cfg = dict2obj(fn.read_json() if fn.exists() else {})
+        val = nested_attr(cfg, attr)
+        if val is None:
+            _nested_setdefault(cfg, attr, hook)
+            fn.write_text(dumps(obj2dict(cfg), indent=2))
+        elif val != hook:
+            sys.stderr.write(f"Can't install hook to '{p}' since it already contains `{attr} = '{val}'`. "
+                             f"Manually update to `{attr} = '{hook}'` for this functionality.")

nbs/11_clean.ipynb

@@ -170,8 +170,8 @@
    "source": [
     "test_nb = read_nb('../tests/metadata.ipynb')\n",
     "\n",
-    "assert set(['meta', 'jekyll', 'my_extra_key', 'my_removed_key']) <= set(test_nb.metadata.keys())\n",
-    "assert set(['meta', 'hide_input', 'my_extra_cell_key', 'my_removed_cell_key']) == set(test_nb.cells[1].metadata.keys())"
+    "assert {'meta', 'jekyll', 'my_extra_key', 'my_removed_key'} <= test_nb.metadata.keys()\n",
+    "assert {'meta', 'hide_input', 'my_extra_cell_key', 'my_removed_cell_key'} == test_nb.cells[1].metadata.keys()"
    ]
   },
   {
@@ -189,8 +189,8 @@
    "source": [
     "clean_nb(test_nb)\n",
     "\n",
-    "assert set(['jekyll', 'kernelspec']) == set(test_nb.metadata.keys())\n",
-    "assert set(['hide_input']) == set(test_nb.cells[1].metadata.keys())"
+    "assert {'jekyll', 'kernelspec'} == test_nb.metadata.keys()\n",
+    "assert {'hide_input'} == test_nb.cells[1].metadata.keys()"
    ]
   },
   {
@@ -209,8 +209,8 @@
     "test_nb = read_nb('../tests/metadata.ipynb')\n",
     "clean_nb(test_nb, allowed_metadata_keys={'my_extra_key'}, allowed_cell_metadata_keys={'my_extra_cell_key'})\n",
     "\n",
-    "assert set(['jekyll', 'kernelspec', 'my_extra_key']) == set(test_nb.metadata.keys())\n",
-    "assert set(['hide_input', 'my_extra_cell_key']) == set(test_nb.cells[1].metadata.keys())"
+    "assert {'jekyll', 'kernelspec', 'my_extra_key'} == test_nb.metadata.keys()\n",
+    "assert {'hide_input', 'my_extra_cell_key'} == test_nb.cells[1].metadata.keys()"
    ]
   },
   {
@@ -229,7 +229,7 @@
     "test_nb = read_nb('../tests/metadata.ipynb')\n",
     "clean_nb(test_nb, clear_all=True)\n",
     "\n",
-    "assert set(['jekyll', 'kernelspec']) == set(test_nb.metadata.keys())\n",
+    "assert {'jekyll', 'kernelspec'} == test_nb.metadata.keys()\n",
     "test_eq(test_nb.cells[1].metadata, {})"
    ]
   },
@@ -272,6 +272,20 @@
     "        warn(e)"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#|export\n",
+    "def _nbdev_clean(nb, **kwargs):\n",
+    "    allowed_metadata_keys = config_key(\"allowed_metadata_keys\", '', missing_ok=True, path=False).split()\n",
+    "    allowed_cell_metadata_keys = config_key(\"allowed_cell_metadata_keys\", '', missing_ok=True, path=False).split()\n",
+    "    return clean_nb(nb, allowed_metadata_keys=allowed_metadata_keys,\n",
+    "                    allowed_cell_metadata_keys=allowed_cell_metadata_keys, **kwargs)"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -288,11 +302,7 @@
     "):\n",
     "    \"Clean all notebooks in `fname` to avoid merge conflicts\"\n",
     "    # Git hooks will pass the notebooks in stdin\n",
-    "    allowed_metadata_keys = config_key(\"allowed_metadata_keys\", default='', missing_ok=True, path=False).split()\n",
-    "    allowed_cell_metadata_keys = config_key(\"allowed_cell_metadata_keys\", default='', missing_ok=True, path=False).split()\n",
-    "    _clean = partial(clean_nb, clear_all=clear_all,\n",
-    "                     allowed_metadata_keys=allowed_metadata_keys,\n",
-    "                     allowed_cell_metadata_keys=allowed_cell_metadata_keys)\n",
+    "    _clean = partial(_nbdev_clean, clear_all=clear_all)\n",
     "    _write = partial(process_write, warn_msg='Failed to clean notebook', proc_nb=_clean)\n",
     "    if stdin: return _write(f_in=sys.stdin, f_out=sys.stdout)\n",
     "    \n",
@@ -357,6 +367,112 @@
     "    (nb_path/'.gitattributes').write_text(\"**/*.ipynb filter=clean-nbs\\n**/*.ipynb diff=ipynb\\n\")"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#|export\n",
+    "def clean_jupyter(path, model, **kwargs):\n",
+    "    \"Clean Jupyter `model` pre save to `path`\"\n",
+    "    get_config.cache_clear() # Reset Jupyter's cache\n",
+    "    try: cfg = get_config(path=path)\n",
+    "    except FileNotFoundError: return\n",
+    "    in_nbdev_repo = 'nbs_path' in cfg\n",
+    "    jupyter_hooks = str2bool(cfg.get('jupyter_hooks', True))\n",
+    "    is_nb_v4 = (model['type'],model['content']['nbformat']) == ('notebook',4)\n",
+    "    if in_nbdev_repo and jupyter_hooks and is_nb_v4: _nbdev_clean(model['content'])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "`clean_jupyter` implements Jupyter's [`ContentsManager.pre_save_hook`](https://jupyter-notebook.readthedocs.io/en/6.4.12/extending/savehooks.html). The easiest way to install it as a Jupyter Notebook or Lab pre-save hook is by running  `nbdev_install_jupyter_hooks`."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#|export\n",
+    "def _nested_setdefault(o, attr, default):\n",
+    "    \"Same as `setdefault`, but if `attr` includes a `.`, then looks inside nested objects\"\n",
+    "    attrs = attr.split('.')\n",
+    "    for a in attrs[:-1]: o = o.setdefault(a, type(o)())\n",
+    "    return o.setdefault(attrs[-1], default)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#|hide\n",
+    "o = {'e':'f'}\n",
+    "test_eq(_nested_setdefault(o, 'a.b.c', 'd'), 'd')\n",
+    "test_eq(o, {'a':{'b':{'c':'d'}},'e':'f'})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#|hide\n",
+    "o = {'a':'b'}\n",
+    "test_eq(_nested_setdefault(o, 'a', 'c'), 'b')\n",
+    "test_eq(o, {'a':'b'})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#|hide\n",
+    "o = {'a':{'b':'c'}}\n",
+    "test_eq(_nested_setdefault(o, 'a.b', 'd'), 'c')\n",
+    "test_eq(o,{'a':{'b':'c'}})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#|export\n",
+    "@call_parse\n",
+    "def nbdev_install_jupyter_hooks():\n",
+    "    \"Install Jupyter hooks to clean notebooks on save\"\n",
+    "    cfg_path = Path.home()/'.jupyter'\n",
+    "    cfg_fns = [cfg_path/f'jupyter_{o}_config.json' for o in ('notebook','server')]\n",
+    "    attr,hook = 'ContentsManager.pre_save_hook','nbdev.clean.clean_jupyter'\n",
+    "    for fn in cfg_fns:\n",
+    "        cfg = dict2obj(fn.read_json() if fn.exists() else {})\n",
+    "        val = nested_attr(cfg, attr)\n",
+    "        if val is None:\n",
+    "            _nested_setdefault(cfg, attr, hook)\n",
+    "            fn.write_text(dumps(obj2dict(cfg), indent=2))\n",
+    "        elif val != hook:\n",
+    "            sys.stderr.write(f\"Can't install hook to '{p}' since it already contains `{attr} = '{val}'`. \"\n",
+    "                             f\"Manually update to `{attr} = '{hook}'` for this functionality.\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "`nbdev`'s Jupyter hooks only run on notebooks in an `nbdev` repo. Hooks can also be disabled at any time by setting `jupyter_hooks = False` in `settings.ini`."
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},

nbs/getting_started.ipynb

@@ -101,7 +101,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "(There's an [alternate version of the walkthru](https://youtu.be/67FdzLSt4aA) avaialable with the coding sections sped up using the `unsilence` python library -- it's 27 minutes faster, but it's be harder to follow along with.)\n",
+    "(There's an [alternate version of the walkthru](https://youtu.be/67FdzLSt4aA) available with the coding sections sped up using the `unsilence` python library -- it's 27 minutes faster, but it's be harder to follow along with.)\n",
     "\n",
     "You can run `nbdev_help` from the terminal to see the full list of available commands:"
    ]

nbs/tutorial.ipynb

@@ -157,14 +157,14 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "## Install git hooks to avoid and handle conflicts"
+    "## Install git and Jupyter hooks for git-friendly notebooks"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "Jupyter Notebooks can cause challenges with git conflicts, but life becomes much easier when you use `nbdev`. As a first step, run `nbdev_install_hooks` in the terminal from your project folder. This will set up hooks which will remove metadata from your notebooks when you commit, greatly reducing the chance you have a conflict.\n",
+    "Jupyter Notebooks store additional metadata (like cell execution order) which cause challenges with git. `nbdev` makes working with notebooks becomes much easier. As a first step, run `nbdev_install_hooks` in the terminal from your project folder. This sets up git hooks which remove metadata from your notebooks automatically, avoiding unnecessary file changes and greatly reducing the chance of a conflict. You can also run `nbdev_install_jupyter_hooks` to install a similar hook that runs on save in Jupyter Notebook and Lab.\n",
     "\n",
     "But if you do get a conflict later, simply run `nbdev_fix filename.ipynb`. This will replace any conflicts in cell outputs with your version, and if there are conflicts in input cells, then both cells will be included in the merged file, along with standard conflict markers (e.g. `=====`). Then you can open the notebook in Jupyter and choose which version to keep."
    ]
@@ -541,7 +541,7 @@
        "Do the saying"
       ],
       "text/plain": [
-       "<nbdev.showdoc.BasicMarkdownRenderer at 0x117e8bf70>"
+       "<nbdev.showdoc.BasicMarkdownRenderer at 0x11595a6d0>"
       ]
      },
      "execution_count": null,

settings.ini

@@ -31,6 +31,7 @@ console_scripts = nbdev_create_config=nbdev.read:nbdev_create_config
 	nbdev_trust=nbdev.clean:nbdev_trust
 	nbdev_clean=nbdev.clean:nbdev_clean
 	nbdev_install_hooks=nbdev.clean:nbdev_install_hooks
+	nbdev_install_jupyter_hooks=nbdev.clean:nbdev_install_jupyter_hooks
 	nbdev_filter=nbdev.cli:nbdev_filter
 	nbdev_quarto=nbdev.cli:nbdev_quarto
 	nbdev_ghp_deploy=nbdev.cli:nbdev_ghp_deploy