Merge pull request #704 from seeM/git-hooks

git merge hooks: automatically resolve conflicts and render markers as separate cells

Author: jph00

nbdev/_modidx.py

@@ -9,6 +9,7 @@
                                    'nbdev_update=nbdev.sync:nbdev_update\n'
                                    'nbdev_export=nbdev.doclinks:nbdev_export\n'
                                    'nbdev_fix=nbdev.merge:nbdev_fix\n'
+                                   'nbdev_merge=nbdev.merge:nbdev_merge\n'
                                    'nbdev_trust=nbdev.clean:nbdev_trust\n'
                                    'nbdev_clean=nbdev.clean:nbdev_clean\n'
                                    'nbdev_install_hooks=nbdev.clean:nbdev_install_hooks\n'
@@ -120,6 +121,7 @@
                              'nbdev.maker.update_var': 'https://nbdev.fast.ai/maker.html#update_var'},
             'nbdev.merge': { 'nbdev.merge.conf_re': 'https://nbdev.fast.ai/merge.html#conf_re',
                              'nbdev.merge.nbdev_fix': 'https://nbdev.fast.ai/merge.html#nbdev_fix',
+                             'nbdev.merge.nbdev_merge': 'https://nbdev.fast.ai/merge.html#nbdev_merge',
                              'nbdev.merge.unpatch': 'https://nbdev.fast.ai/merge.html#unpatch'},
             'nbdev.migrate': { 'nbdev.migrate.migrate_md_fm': 'https://nbdev.fast.ai/migrate.html#migrate_md_fm',
                                'nbdev.migrate.migrate_nb_fm': 'https://nbdev.fast.ai/migrate.html#migrate_nb_fm',

nbdev/clean.py

@@ -140,8 +140,9 @@ def _nested_setdefault(o, attr, default):
 # %% ../nbs/11_clean.ipynb 30
 @call_parse
 def nbdev_install_hooks():
-    "Install Jupyter and git hooks to clean notebooks on save and trust notebooks automatically"
+    "Install Jupyter and git hooks to automatically clean, trust, and fix merge conflicts in notebooks"
     cfg_path = Path.home()/'.jupyter'
+    cfg_path.mkdir(exist_ok=True)
     cfg_fns = [cfg_path/f'jupyter_{o}_config.json' for o in ('notebook','server')]
     attr,hook = 'ContentsManager.pre_save_hook','nbdev.clean.clean_jupyter'
     for fn in cfg_fns:
@@ -161,4 +162,21 @@ def nbdev_install_hooks():
     hook_path.mkdir(parents=True, exist_ok=True)
     fn.write_text("#!/bin/bash\nnbdev_trust")
     os.chmod(fn, os.stat(fn).st_mode | stat.S_IEXEC)
+
+    (path/'.gitconfig').write_text('''# Generated by nbdev_install_hooks
+#
+# If you need to disable this instrumentation do:
+#   git config --local --unset include.path
+#
+# To restore:
+#   git config --local include.path .gitconfig
+#
+[merge "nbdev-merge"]
+	name = resolve conflicts with nbdev_fix
+	driver = nbdev_merge %O %A %B %P
+''')
+    cmd = "git config --local include.path ../.gitconfig"
+    run(cmd)
+    (nb_path/'.gitattributes').write_text("*.ipynb merge=nbdev-merge\n")
+
     print("Hooks are installed.")

nbdev/merge.py

@@ -1,7 +1,7 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/06_merge.ipynb.
 
 # %% auto 0
-__all__ = ['conf_re', 'unpatch', 'nbdev_fix']
+__all__ = ['conf_re', 'nbdev_fix', 'unpatch', 'nbdev_merge']
 
 # %% ../nbs/06_merge.ipynb 2
 from .imports import *
@@ -13,6 +13,7 @@
 from fastcore.script import *
 from fastcore import shutil
 
+import subprocess
 from difflib import SequenceMatcher
 
 # %% ../nbs/06_merge.ipynb 16
@@ -53,8 +54,7 @@ def _merge_cells(a, b, brancha, branchb, theirs):
     return res,conflict
 
 # %% ../nbs/06_merge.ipynb 23
-@call_parse
-def nbdev_fix(nbname:str, # Notebook filename to fix
+def _nbdev_fix(nbname:str, # Notebook filename to fix
               outname:str=None, # Filename of output notebook (defaults to `nbname`)
               nobackup:bool=True, # Do not backup `nbname` to `nbname`.bak if `outname` not provided
               theirs:bool=False, # Use their outputs and metadata instead of ours
@@ -73,3 +73,29 @@ def nbdev_fix(nbname:str, # Notebook filename to fix
         if conflict: print("One or more conflict remains in the notebook, please inspect manually.")
         else: print("Successfully merged conflicts!")
     return conflict
+
+# %% ../nbs/06_merge.ipynb 24
+nbdev_fix = call_parse(_nbdev_fix)
+
+# %% ../nbs/06_merge.ipynb 28
+def _only(o):
+    "Return the only item of `o`, raise if `o` doesn't have exactly one item"
+    it = iter(o)
+    try: res = next(it)
+    except StopIteration: raise ValueError('iterable has 0 items') from None
+    try: next(it)
+    except StopIteration: return res
+    raise ValueError(f'iterable has more than 1 item')
+
+# %% ../nbs/06_merge.ipynb 30
+def _git_branch_merge(): return _only(v for k,v in os.environ.items() if k.startswith('GITHEAD'))
+
+# %% ../nbs/06_merge.ipynb 31
+@call_parse
+def nbdev_merge(base:str, ours:str, theirs:str, path:str):
+    "Git merge driver for notebooks"
+    proc = subprocess.run(f'git merge-file -L HEAD -L BASE -L {_git_branch_merge()} {ours} {base} {theirs}',
+                          shell=True, capture_output=True, text=True)
+    if proc.returncode == 0: return
+    theirs = str2bool(os.environ.get('THEIRS', False))
+    return _nbdev_fix(ours, theirs=theirs)

nbs/06_merge.ipynb

@@ -36,6 +36,7 @@
     "from fastcore.script import *\n",
     "from fastcore import shutil\n",
     "\n",
+    "import subprocess\n",
     "from difflib import SequenceMatcher"
    ]
   },
@@ -405,8 +406,7 @@
    "outputs": [],
    "source": [
     "#|export\n",
-    "@call_parse\n",
-    "def nbdev_fix(nbname:str, # Notebook filename to fix\n",
+    "def _nbdev_fix(nbname:str, # Notebook filename to fix\n",
     "              outname:str=None, # Filename of output notebook (defaults to `nbname`)\n",
     "              nobackup:bool=True, # Do not backup `nbname` to `nbname`.bak if `outname` not provided\n",
     "              theirs:bool=False, # Use their outputs and metadata instead of ours\n",
@@ -427,6 +427,16 @@
     "    return conflict"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#|export\n",
+    "nbdev_fix = call_parse(_nbdev_fix)"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -456,6 +466,85 @@
     "os.unlink('tmp.ipynb')"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Git merge driver"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#|export\n",
+    "def _only(o):\n",
+    "    \"Return the only item of `o`, raise if `o` doesn't have exactly one item\"\n",
+    "    it = iter(o)\n",
+    "    try: res = next(it)\n",
+    "    except StopIteration: raise ValueError('iterable has 0 items') from None\n",
+    "    try: next(it)\n",
+    "    except StopIteration: return res\n",
+    "    raise ValueError(f'iterable has more than 1 item')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#|hide\n",
+    "test_fail(lambda: _only([]), contains='iterable has 0 items')\n",
+    "test_eq(_only([0]), 0)\n",
+    "test_fail(lambda: _only([0,1]), contains='iterable has more than 1 item')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#|export\n",
+    "def _git_branch_merge(): return _only(v for k,v in os.environ.items() if k.startswith('GITHEAD'))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#|export\n",
+    "@call_parse\n",
+    "def nbdev_merge(base:str, ours:str, theirs:str, path:str):\n",
+    "    \"Git merge driver for notebooks\"\n",
+    "    proc = subprocess.run(f'git merge-file -L HEAD -L BASE -L {_git_branch_merge()} {ours} {base} {theirs}',\n",
+    "                          shell=True, capture_output=True, text=True)\n",
+    "    if proc.returncode == 0: return\n",
+    "    theirs = str2bool(os.environ.get('THEIRS', False))\n",
+    "    return _nbdev_fix(ours, theirs=theirs)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "This implements a [git merge driver](https://git-scm.com/docs/gitattributes#_defining_a_custom_merge_driver) for notebooks that automatically resolves conflicting metadata and outputs, and splits remaining conflicts as separate cells so that the notebook can be viewed and fixed in Jupyter. The easiest way to install it is by running `nbdev_install_hooks`."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "This works by first running Git's default merge driver, and then `nbdev_fix` if there are still conflicts. You can set `nbdev_fix`'s `theirs` argument using the `THEIRS` environment variable, for example:\n",
+    "\n",
+    "    THEIRS=True git merge branch"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},