Skip to content

fix: duplication of directives with nbdev_update #1530 #1531

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix: duplication of directives with nbdev_update #1530 #1531

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions nbdev/test_update_directives.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
# AUTOGENERATED! DO NOT EDIT! File to edit: ../tests/update_directives.ipynb.

# %% auto 0
__all__ = []

# %% ../tests/update_directives.ipynb 5
#| output: False
"""
Think of something
that makes you smile
and brightens your day
"""

# System import
import os
# Local import
import nbdev.serve

# %% ../tests/update_directives.ipynb 6
# |eval: false
print("1. test export and eval")

# %% ../tests/update_directives.ipynb 8
#| eval: false
print("3. nr 2 should be missing from the py file")
142 changes: 141 additions & 1 deletion nbs/api/06_sync.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,10 +154,150 @@
" assert cell.nb_path == nb_path\n",
" nbcell = nbp.nb.cells[cell.idx]\n",
" dirs,_ = _partition_cell(nbcell, 'python')\n",
" nbcell.source = ''.join(dirs) + _to_absolute(cell.code, cell.py_path, lib_dir)\n",
" prelim = _to_absolute(cell.code, cell.py_path, lib_dir)\n",
" for i in set(dirs):\n",
" if '#|' in i.replace(' ',''):\n",
" prelim=prelim.replace(i, '')\n",
" final = ''.join(set(dirs)) + prelim\n",
" nbcell.source = final\n",
" write_nb(nbp.nb, nb_path)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# (Re)Define functions with and without patch\n",
"\n",
"# copy/paste of _update_nb before patch was applied\n",
"def _update_nb_without_patch(nb_path, cells, lib_dir):\n",
" \"Update notebook `nb_path` with contents from `cells`\"\n",
" nbp = NBProcessor(nb_path, ExportModuleProc(), rm_directives=False)\n",
" nbp.process()\n",
" for cell in cells:\n",
" assert cell.nb_path == nb_path\n",
" nbcell = nbp.nb.cells[cell.idx]\n",
" dirs,_ = _partition_cell(nbcell, 'python')\n",
" nbcell.source = ''.join(dirs) + _to_absolute(cell.code, cell.py_path, lib_dir)\n",
" write_nb(nbp.nb, nb_path)\n",
"\n",
"\n",
"def _update_nb_with_patch_and_prints(nb_path, cells, lib_dir):\n",
" \"Update notebook `nb_path` with contents from `cells`\"\n",
" nbp = NBProcessor(nb_path, ExportModuleProc(), rm_directives=False)\n",
" nbp.process()\n",
" for cell in cells:\n",
" # print('\\n********cell*********')\n",
" assert cell.nb_path == nb_path\n",
" nbcell = nbp.nb.cells[cell.idx]\n",
" dirs,_ = _partition_cell(nbcell, 'python')\n",
" prelim = _to_absolute(cell.code, cell.py_path, lib_dir)\n",
" # print('\\n>>>>>>>prelim:\\n\\n', prelim)\n",
" for i in set(dirs):\n",
" if '#|' in i.replace(' ',''):\n",
" # print('to replace', i)\n",
" prelim=prelim.replace(i, '') \n",
" # print('\\n>>>>>>>inter\\n\\n', prelim)\n",
" final = ''.join(set(dirs)) + prelim\n",
" # print('\\n>>>>>>>final\\n\\n', final)\n",
" nbcell.source = final\n",
" write_nb(nbp.nb, nb_path)\n",
"\n",
"\n",
"# based on value of `with_patch`, a different version of _update_nb will be used\n",
"def _update_mod_patch(py_path, lib_dir, with_patch):\n",
" \"Propagate changes from cells in module `py_path` to corresponding notebooks\"\n",
" py_cells = L(_iter_py_cells(py_path)).filter(lambda o: o.nb != 'auto')\n",
" if with_patch:\n",
" nb_function=_update_nb_with_patch_and_prints\n",
" else:\n",
" nb_function=_update_nb_without_patch\n",
" for nb_path,cells in groupby(py_cells, 'nb_path').items(): nb_function(nb_path, cells, lib_dir)\n",
"\n",
"\n",
"# based on value of `with_patch`, a different version of `_update_mod` will be used\n",
"@call_parse\n",
"def nbdev_update_patch(fname:str=None, with_patch=True): # A Python file name to update\n",
" \"Propagate change in modules matching `fname` to notebooks that created them\"\n",
" if fname and fname.endswith('.ipynb'): raise ValueError(\"`nbdev_update` operates on .py files. If you wish to convert notebooks instead, see `nbdev_export`.\")\n",
" if os.environ.get('IN_TEST',0): return\n",
" cfg = get_config()\n",
" if not cfg.cell_number: raise ValueError(\"`nbdev_update` does not support without cell_number in .py files. Please check your settings.ini\")\n",
" fname = Path(fname or cfg.lib_path)\n",
" lib_dir = cfg.lib_path.parent\n",
" files = globtastic(fname, file_glob='*.py', skip_folder_re='^[_.]').filter(lambda x: str(Path(x).absolute().relative_to(lib_dir) in _mod_files()))\n",
" files.map(_update_mod_patch, with_patch=with_patch, lib_dir=lib_dir)\n",
"\n",
"\n",
"def _update_py_file(f='../../nbdev/test_update_directives.py', restore=False):\n",
" \"\"\" adds or removes line from py file \"\"\"\n",
" to_append = \"\\nprint('add new line')\\n\"\n",
"\n",
" if restore is False:\n",
" with open(f, 'a') as file:\n",
" file.write(to_append)\n",
" else:\n",
" with open(f, 'r') as file:\n",
" s = file.read()\n",
" upd = s.replace(to_append, '')\n",
" \n",
" with open(f, 'w') as file:\n",
" file.write(upd)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"py_file = '../../nbdev/test_update_directives.py'\n",
"ipynb_file = '../../tests/update_directives.ipynb'\n",
"\n",
"# read original state of pyfile\n",
"with open(py_file, 'r') as file:\n",
" py_before_update = file.read()\n",
"\n",
"# modify py_file\n",
"_update_py_file(restore=False)\n",
"\n",
"# read updated pyfile\n",
"with open(py_file, 'r') as file:\n",
" py_after_update = file.read()\n",
"\n",
"assert py_before_update != py_after_update \n",
"\n",
"# read original state of ipynb \n",
"with open(ipynb_file, 'r') as file:\n",
" nbs_before_update = file.read()\n",
"\n",
"# update ipynb without patch\n",
"nbdev_update_patch(fname=py_file, with_patch=False)\n",
"# read updated ipynb\n",
"with open(ipynb_file, 'r') as file:\n",
" nbs_after_update_without_patch = file.read()\n",
"\n",
"# restore py file\n",
"_update_py_file(restore=True)\n",
"\n",
"# update py file\n",
"_update_py_file(restore=False)\n",
"# update ipynb without patch\n",
"nbdev_update_patch(fname=py_file, with_patch=True)\n",
"# read updated ipynb\n",
"with open(ipynb_file, 'r') as file:\n",
" nbs_after_update_with_patch = file.read()\n",
"\n",
"# restore py file\n",
"_update_py_file(restore=True)\n",
"\n",
"assert nbs_after_update_without_patch != nbs_after_update_with_patch \n",
"assert \"\"\"\"#| eval: false\\\\n\",\\n \"#| eval: false\\\\n\",\\n\"\"\" not in nbs_after_update_with_patch, 'duplicated directive in version with the patch' \n",
"assert \"\"\"\"#| eval: false\\\\n\",\\n \"#| eval: false\\\\n\",\\n\"\"\" not in nbs_after_update_without_patch, 'duplicated directive in version without the patch'"
]
},
{
"cell_type": "code",
"execution_count": null,
Expand Down
159 changes: 159 additions & 0 deletions tests/update_directives.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,159 @@
{
"cells": [
{
"cell_type": "raw",
"id": "3e3b86e8-8483-4036-950f-e7141b0e01ab",
"metadata": {},
"source": [
"---\n",
"skip_showdocs: True\n",
"---"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a57c5c95-9e94-4462-a4a4-c254812f592c",
"metadata": {},
"outputs": [],
"source": [
"#|default_exp test_update_directives"
]
},
{
"attachments": {},
"cell_type": "markdown",
"id": "ca768512-0a56-4310-954c-82ca90061e62",
"metadata": {},
"source": [
"# Test sync.py\n",
"\n",
"### This notebook can be used to test nbdev_update. \n",
"\n",
"#### ! It should not be exported ! \n",
"otherwise the cell numbers will be removed from the py file, and further testing of nbdev_update won't be possible. "
]
},
{
"cell_type": "markdown",
"id": "0ebb900e-faf5-417b-8557-70062f225130",
"metadata": {},
"source": [
"## How to use\n",
"### manually\n",
"- edit nbdev/test_update_directives.py and save it \n",
"- head over to nbs/api/06_sync.ipynb \n",
"- modify one of the functions, eg. _update_nb \n",
"- execute in the sync notebook: \n",
"\n",
"```\n",
"nbdev_update('../../nbdev/test_directives.py')\n",
"```\n",
"\n",
"- and see if this notebook was modified as expected\n",
"\n",
"### tests in nbs/api/06_sync.ipynb\n",
"- use the section with (re)definition of functions and the tests that follow them "
]
},
{
"cell_type": "markdown",
"id": "d9ce75fd-8f51-4a61-aa42-2c9bcd8d70d7",
"metadata": {},
"source": [
"The following cells also test various combinations of spaces in '#|'"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "238a3c7e-4ff4-4e6f-8377-f2ec41bdfdb6",
"metadata": {},
"outputs": [],
"source": [
"# | export\n",
"#| output: False\n",
"\n",
"\"\"\"\n",
"Think of something\n",
"that makes you smile \n",
"and brightens your day\n",
"\"\"\"\n",
"\n",
"# System import\n",
"import os\n",
"# Local import\n",
"import nbdev.serve"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e4175dcd-2b45-4a1e-971b-c6a43e83d31f",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"test export and eval\n"
]
}
],
"source": [
"# |exports\n",
"# |eval: false\n",
"print(\"1. test export and eval\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d3870aff-7673-4e19-9dc6-a422e1a9aa20",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"more cells :D. Let's test eval and hide\n"
]
}
],
"source": [
"#| eval: false\n",
"#| hide\n",
"print(\"2. more cells :D. Let's test eval and hide\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "117a87dc-60ed-486e-9f11-a4f24ed7d5b5",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"3. nr 2 should be missing from the py file\n"
]
}
],
"source": [
"#| eval: false\n",
"#| export\n",
"print(\"3. nr 2 should be missing from the py file\")"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "python3",
"language": "python",
"name": "python3"
}
},
"nbformat": 4,
"nbformat_minor": 5
}