From cb98b2eb6b7b9c09761da0108930bb77aee26f8c Mon Sep 17 00:00:00 2001 From: Mahendra Paipuri Date: Sun, 18 Feb 2024 10:35:46 +0100 Subject: [PATCH 1/2] test: Use current env for pre-commit tests * Use current test env to run pre-commit tests * This avoids creating a new env and installing jupytext in that env Signed-off-by: Mahendra Paipuri --- tests/conftest.py | 33 ++++++++++ .../test_pre_commit_0_ipynb_to_py.py | 31 +++++---- .../pre_commit/test_pre_commit_1_sync.py | 16 ++--- .../test_pre_commit_1_sync_with_config.py | 16 ++--- .../test_pre_commit_1_sync_with_no_config.py | 26 ++++---- .../test_pre_commit_2_sync_nbstripout.py | 28 ++++---- ...test_pre_commit_3_sync_black_nbstripout.py | 50 ++++++++------ .../test_pre_commit_4_sync_execute.py | 28 ++++---- .../test_pre_commit_5_reformat_markdown.py | 65 +++++++++++++------ 9 files changed, 184 insertions(+), 109 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 3d76dfbfb..3341c5db3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,6 +6,7 @@ from pathlib import Path import pytest +import yaml from jupyter_client.kernelspec import find_kernel_specs, get_kernel_spec from nbformat.v4 import nbbase from nbformat.v4.nbbase import ( @@ -63,6 +64,38 @@ def jupytext_repo_rev(jupytext_repo_root): return system("git", "rev-parse", "HEAD", cwd=jupytext_repo_root).strip() +@pytest.fixture +def jupytext_pre_commit_config(jupytext_repo_root): + """The local revision of this repo, to use in .pre-commit-config.yaml in tests""" + # Read pre-commit-hooks.yaml file + # + # Setting language to system will ensure that pre-commit assumes that we + # provision correct environment. This is the case when we run unit tests as we + # make a developmental install of jupytext before running unit tests. So there is + # an "test" environment that is provisioned and we use this current environment + # to run pre-commit tests. + # + # When there are additional_dependencies, we override this in individual test to + # python so that pre-commit will create an environment and install those + # additional dependencies in that environment. + # + # This strategy will enable us to directly test the pre-commit hook config with + # current version of jupytext. It also avoid re-installing jupytext in pre-commit + # config and thus tests run faster. + with open(os.path.join(jupytext_repo_root, ".pre-commit-hooks.yaml")) as file: + pre_commit_hooks = yaml.safe_load(file) + pre_commit_hooks[0]["language"] = "system" + pre_commit_config = { + "repos": [ + { + "repo": "local", + "hooks": pre_commit_hooks, + } + ] + } + return pre_commit_config + + @pytest.fixture() def python_notebook(): return new_notebook( diff --git a/tests/external/pre_commit/test_pre_commit_0_ipynb_to_py.py b/tests/external/pre_commit/test_pre_commit_0_ipynb_to_py.py index 2b3af5a8f..155c986eb 100644 --- a/tests/external/pre_commit/test_pre_commit_0_ipynb_to_py.py +++ b/tests/external/pre_commit/test_pre_commit_0_ipynb_to_py.py @@ -1,4 +1,7 @@ +import os + import pytest +import yaml from git.exc import HookExecutionError from nbformat.v4.nbbase import new_markdown_cell, new_notebook from pre_commit.main import main as pre_commit @@ -9,25 +12,31 @@ def test_pre_commit_hook_ipynb_to_py( - tmpdir, cwd_tmpdir, tmp_repo, jupytext_repo_root, jupytext_repo_rev + tmpdir, + cwd_tmpdir, + tmp_repo, + jupytext_repo_root, + jupytext_repo_rev, + jupytext_pre_commit_config, ): """Here we document and test the expected behavior of the pre-commit hook in the directional (--to) mode. Note that here, the ipynb file is always the source for updates - i.e. changes on the .py file will not trigger the hook. """ # set up the tmpdir repo with pre-commit - pre_commit_config_yaml = f""" -repos: -- repo: {jupytext_repo_root} - rev: {jupytext_repo_rev} - hooks: - - id: jupytext - args: [--from, ipynb, --to, "py:percent"] -""" + # Add args as if we will add in actual .pre-commit-config.yaml file + jupytext_pre_commit_config["repos"][0]["hooks"][0]["args"] = [ + "--from", + "ipynb", + "--to", + "py:percent", + ] + with open(os.path.join(tmpdir, ".pre-commit-config.yaml"), "w") as file: + yaml.dump(jupytext_pre_commit_config, file) - tmpdir.join(".pre-commit-config.yaml").write(pre_commit_config_yaml) + # tmpdir.join(".pre-commit-config.yaml").write(pre_commit_config_yaml) tmp_repo.git.add(".pre-commit-config.yaml") - pre_commit(["install", "--install-hooks"]) + pre_commit(["install", "--install-hooks", "-f"]) # write test notebook and output file nb = new_notebook(cells=[new_markdown_cell("A short notebook")]) diff --git a/tests/external/pre_commit/test_pre_commit_1_sync.py b/tests/external/pre_commit/test_pre_commit_1_sync.py index 6b9854040..941736fe2 100644 --- a/tests/external/pre_commit/test_pre_commit_1_sync.py +++ b/tests/external/pre_commit/test_pre_commit_1_sync.py @@ -1,6 +1,8 @@ +import os import shutil import pytest +import yaml from git.exc import HookExecutionError from nbformat.v4.nbbase import new_markdown_cell from pre_commit.main import main as pre_commit @@ -15,17 +17,13 @@ def test_pre_commit_hook_sync( tmp_repo, jupytext_repo_root, jupytext_repo_rev, + jupytext_pre_commit_config, python_notebook, ): - pre_commit_config_yaml = f""" -repos: -- repo: {jupytext_repo_root} - rev: {jupytext_repo_rev} - hooks: - - id: jupytext - args: [--sync] -""" - tmpdir.join(".pre-commit-config.yaml").write(pre_commit_config_yaml) + # Add args as if we will add in actual .pre-commit-config.yaml file + jupytext_pre_commit_config["repos"][0]["hooks"][0]["args"] = ["--sync"] + with open(os.path.join(tmpdir, ".pre-commit-config.yaml"), "w") as file: + yaml.dump(jupytext_pre_commit_config, file) tmp_repo.git.add(".pre-commit-config.yaml") pre_commit(["install", "--install-hooks", "-f"]) diff --git a/tests/external/pre_commit/test_pre_commit_1_sync_with_config.py b/tests/external/pre_commit/test_pre_commit_1_sync_with_config.py index 1ef521cff..009578647 100644 --- a/tests/external/pre_commit/test_pre_commit_1_sync_with_config.py +++ b/tests/external/pre_commit/test_pre_commit_1_sync_with_config.py @@ -1,4 +1,7 @@ +import os + import pytest +import yaml from git.exc import HookExecutionError from nbformat.v4.nbbase import new_markdown_cell from pre_commit.main import main as pre_commit @@ -12,17 +15,12 @@ def test_pre_commit_hook_sync_with_config( tmp_repo, jupytext_repo_root, jupytext_repo_rev, + jupytext_pre_commit_config, python_notebook, ): - pre_commit_config_yaml = f""" -repos: -- repo: {jupytext_repo_root} - rev: {jupytext_repo_rev} - hooks: - - id: jupytext - args: [--sync] -""" - tmpdir.join(".pre-commit-config.yaml").write(pre_commit_config_yaml) + jupytext_pre_commit_config["repos"][0]["hooks"][0]["args"] = ["--sync"] + with open(os.path.join(tmpdir, ".pre-commit-config.yaml"), "w") as file: + yaml.dump(jupytext_pre_commit_config, file) tmp_repo.git.add(".pre-commit-config.yaml") pre_commit(["install", "--install-hooks", "-f"]) diff --git a/tests/external/pre_commit/test_pre_commit_1_sync_with_no_config.py b/tests/external/pre_commit/test_pre_commit_1_sync_with_no_config.py index 4222cc91f..24dfb5ef6 100644 --- a/tests/external/pre_commit/test_pre_commit_1_sync_with_no_config.py +++ b/tests/external/pre_commit/test_pre_commit_1_sync_with_no_config.py @@ -1,6 +1,8 @@ +import os from copy import deepcopy import pytest +import yaml from git.exc import HookExecutionError from nbformat.v4.nbbase import new_markdown_cell from pre_commit.main import main as pre_commit @@ -16,23 +18,18 @@ def test_pre_commit_hook_sync_with_no_config( tmp_repo, jupytext_repo_root, jupytext_repo_rev, + jupytext_pre_commit_config, notebook_with_outputs, ): """In this test we reproduce the setting from https://github.com/mwouts/jupytext/issues/967""" - pre_commit_config_yaml = f""" -repos: -- repo: {jupytext_repo_root} - rev: {jupytext_repo_rev} - hooks: - - id: jupytext - args: [ - '--sync', - '--set-formats', - 'ipynb,py:percent', - '--show-changes', - '--' + jupytext_pre_commit_config["repos"][0]["hooks"][0]["args"] = [ + "--sync", + "--set-formats", + "ipynb,py:percent", + "--show-changes", + "--", ] -""" + # Save a sample notebook with outputs in Jupyter nb = deepcopy(notebook_with_outputs) (tmpdir / "notebooks").mkdir() @@ -45,7 +42,8 @@ def test_pre_commit_hook_sync_with_no_config( tmp_repo.index.commit("Notebook with outputs") # Configure the pre-commit hook - tmpdir.join(".pre-commit-config.yaml").write(pre_commit_config_yaml) + with open(os.path.join(tmpdir, ".pre-commit-config.yaml"), "w") as file: + yaml.dump(jupytext_pre_commit_config, file) tmp_repo.git.add(".pre-commit-config.yaml") pre_commit(["install", "--install-hooks", "-f"]) diff --git a/tests/external/pre_commit/test_pre_commit_2_sync_nbstripout.py b/tests/external/pre_commit/test_pre_commit_2_sync_nbstripout.py index 304a84f5b..49b1ea24a 100644 --- a/tests/external/pre_commit/test_pre_commit_2_sync_nbstripout.py +++ b/tests/external/pre_commit/test_pre_commit_2_sync_nbstripout.py @@ -1,4 +1,7 @@ +import os + import pytest +import yaml from git.exc import HookExecutionError from pre_commit.main import main as pre_commit @@ -12,24 +15,21 @@ def test_pre_commit_hook_sync_nbstripout( tmp_repo, jupytext_repo_root, jupytext_repo_rev, + jupytext_pre_commit_config, notebook_with_outputs, ): """Here we sync the ipynb notebook with a Markdown file and also apply nbstripout.""" - pre_commit_config_yaml = f""" -repos: -- repo: {jupytext_repo_root} - rev: {jupytext_repo_rev} - hooks: - - id: jupytext - args: [--sync] - -- repo: https://github.com/kynan/nbstripout - rev: 0.5.0 - hooks: - - id: nbstripout -""" + jupytext_pre_commit_config["repos"][0]["hooks"][0]["args"] = ["--sync"] + jupytext_pre_commit_config["repos"].append( + { + "repo": "https://github.com/kynan/nbstripout", + "rev": "0.5.0", + "hooks": [{"id": "nbstripout"}], + } + ) - tmpdir.join(".pre-commit-config.yaml").write(pre_commit_config_yaml) + with open(os.path.join(tmpdir, ".pre-commit-config.yaml"), "w") as file: + yaml.dump(jupytext_pre_commit_config, file) tmp_repo.git.add(".pre-commit-config.yaml") pre_commit(["install", "--install-hooks", "-f"]) diff --git a/tests/external/pre_commit/test_pre_commit_3_sync_black_nbstripout.py b/tests/external/pre_commit/test_pre_commit_3_sync_black_nbstripout.py index 01a549fec..4cfe2d4cb 100644 --- a/tests/external/pre_commit/test_pre_commit_3_sync_black_nbstripout.py +++ b/tests/external/pre_commit/test_pre_commit_3_sync_black_nbstripout.py @@ -1,4 +1,7 @@ +import os + import pytest +import yaml from git.exc import HookExecutionError from pre_commit.main import main as pre_commit @@ -11,30 +14,37 @@ def test_pre_commit_hook_sync_black_nbstripout( tmp_repo, jupytext_repo_root, jupytext_repo_rev, + jupytext_pre_commit_config, notebook_with_outputs, ): """Here we sync the ipynb notebook with a py:percent file and also apply black and nbstripout.""" - pre_commit_config_yaml = f""" -repos: -- repo: {jupytext_repo_root} - rev: {jupytext_repo_rev} - hooks: - - id: jupytext - args: [--sync, --pipe, black] - additional_dependencies: - - black==22.3.0 # Matches hook - -- repo: https://github.com/psf/black - rev: 22.3.0 - hooks: - - id: black + jupytext_pre_commit_config["repos"][0]["hooks"][0]["args"] = [ + "--sync", + "--pipe", + "black", + ] + jupytext_pre_commit_config["repos"][0]["hooks"][0]["additional_dependencies"] = [ + "black==22.3.0", + ] + # Use python as language as we will need to install additional dependencies + jupytext_pre_commit_config["repos"][0]["hooks"][0]["language"] = "python" + jupytext_pre_commit_config["repos"].append( + { + "repo": "https://github.com/kynan/nbstripout", + "rev": "0.5.0", + "hooks": [{"id": "nbstripout"}], + } + ) + jupytext_pre_commit_config["repos"].append( + { + "repo": "https://github.com/psf/black", + "rev": "22.3.0", + "hooks": [{"id": "black"}], + } + ) -- repo: https://github.com/kynan/nbstripout - rev: 0.5.0 - hooks: - - id: nbstripout -""" - tmpdir.join(".pre-commit-config.yaml").write(pre_commit_config_yaml) + with open(os.path.join(tmpdir, ".pre-commit-config.yaml"), "w") as file: + yaml.dump(jupytext_pre_commit_config, file) tmp_repo.git.add(".pre-commit-config.yaml") pre_commit(["install", "--install-hooks", "-f"]) diff --git a/tests/external/pre_commit/test_pre_commit_4_sync_execute.py b/tests/external/pre_commit/test_pre_commit_4_sync_execute.py index 92694b29b..0be64e6d5 100644 --- a/tests/external/pre_commit/test_pre_commit_4_sync_execute.py +++ b/tests/external/pre_commit/test_pre_commit_4_sync_execute.py @@ -1,4 +1,7 @@ +import os + import pytest +import yaml from git.exc import HookExecutionError from nbformat.v4.nbbase import new_code_cell from pre_commit.main import main as pre_commit @@ -13,21 +16,24 @@ def test_pre_commit_hook_sync_execute( tmp_repo, jupytext_repo_root, jupytext_repo_rev, + jupytext_pre_commit_config, notebook_with_outputs, ): """Here we sync the ipynb notebook with a py:percent file and execute it (this is probably not a very recommendable hook!)""" - pre_commit_config_yaml = f""" -repos: -- repo: {jupytext_repo_root} - rev: {jupytext_repo_rev} - hooks: - - id: jupytext - args: [--sync, --execute, --show-changes] - additional_dependencies: - - nbconvert -""" - tmpdir.join(".pre-commit-config.yaml").write(pre_commit_config_yaml) + jupytext_pre_commit_config["repos"][0]["hooks"][0]["args"] = [ + "--sync", + "--execute", + "--show-changes", + ] + jupytext_pre_commit_config["repos"][0]["hooks"][0]["additional_dependencies"] = [ + "nbconvert" + ] + # Use python as language as we will need to install additional dependencies + jupytext_pre_commit_config["repos"][0]["hooks"][0]["language"] = "python" + + with open(os.path.join(tmpdir, ".pre-commit-config.yaml"), "w") as file: + yaml.dump(jupytext_pre_commit_config, file) tmp_repo.git.add(".pre-commit-config.yaml") pre_commit(["install", "--install-hooks", "-f"]) diff --git a/tests/external/pre_commit/test_pre_commit_5_reformat_markdown.py b/tests/external/pre_commit/test_pre_commit_5_reformat_markdown.py index e55654dbc..d52c4865a 100644 --- a/tests/external/pre_commit/test_pre_commit_5_reformat_markdown.py +++ b/tests/external/pre_commit/test_pre_commit_5_reformat_markdown.py @@ -1,4 +1,7 @@ +import os + import pytest +import yaml from git.exc import HookExecutionError from nbformat.v4.nbbase import new_code_cell, new_markdown_cell, new_notebook from pre_commit.main import main as pre_commit @@ -13,6 +16,7 @@ def test_pre_commit_hook_sync_reformat_code_and_markdown( tmp_repo, jupytext_repo_root, jupytext_repo_rev, + jupytext_pre_commit_config, notebook_with_outputs, ): """Here we sync the ipynb notebook with a py:percent file and also apply black and pandoc to reformat both @@ -22,27 +26,46 @@ def test_pre_commit_hook_sync_reformat_code_and_markdown( ipynb files. Consequently we pin the version of nbformat to 5.0.8 in all the environments below (and you will have to do the same on the environment in which you edit the notebooks). """ - pre_commit_config_yaml = f""" -repos: -- repo: {jupytext_repo_root} - rev: {jupytext_repo_rev} - hooks: - - id: jupytext - args: [--sync, --pipe-fmt, ipynb, --pipe, 'pandoc --from ipynb --to ipynb --markdown-headings=atx', --show-changes] - additional_dependencies: - - nbformat==5.0.8 # because pandoc 2.11.4 does not preserve yet the new cell ids - - id: jupytext - args: [--sync, --pipe, black, --show-changes] - additional_dependencies: - - black==22.3.0 # Matches black hook below - - nbformat==5.0.8 # for compatibility with the pandoc hook above - -- repo: https://github.com/psf/black - rev: 22.3.0 - hooks: - - id: black -""" - tmpdir.join(".pre-commit-config.yaml").write(pre_commit_config_yaml) + jupytext_pre_commit_config["repos"][0]["hooks"][0]["args"] = [ + "--sync", + "--pipe-fmt", + "ipynb", + "--pipe", + "pandoc", + "--from", + "ipynb", + "--to", + "ipynb", + "--markdown-headings=atx", + "--show-changes", + ] + jupytext_pre_commit_config["repos"][0]["hooks"][0]["additional_dependencies"] = [ + "nbconvert==5.0.8" + ] + jupytext_pre_commit_config["repos"][0]["hooks"][1] = jupytext_pre_commit_config[ + "repos" + ][0]["hooks"][0] + jupytext_pre_commit_config["repos"][0]["hooks"][1]["args"] = [ + "--sync", + "--pipe", + "black", + "--show-changes", + ] + jupytext_pre_commit_config["repos"][0]["hooks"][1]["additional_dependencies"] = [ + "black==22.3.0", # Matches black hook below + "nbformat==5.0.8", # for compatibility with the pandoc hook above + ] + # Use python as language as we will need to install additional dependencies + jupytext_pre_commit_config["repos"][0]["hooks"][0]["language"] = "python" + jupytext_pre_commit_config["repos"][0]["hooks"][1]["language"] = "python" + jupytext_pre_commit_config["repos"][1] = { + "repo": "https://github.com/psf/black", + "rev": "22.3.0", + "hooks": [{"id": "black"}], + } + + with open(os.path.join(tmpdir, ".pre-commit-config.yaml"), "w") as file: + yaml.dump(jupytext_pre_commit_config, file) tmp_repo.git.add(".pre-commit-config.yaml") pre_commit(["install", "--install-hooks", "-f"]) From 4c1273b10ba3021161e8d1161f9380e51cb2b9d3 Mon Sep 17 00:00:00 2001 From: Mahendra Paipuri Date: Sun, 18 Feb 2024 15:13:41 +0100 Subject: [PATCH 2/2] test: Fix indexError in pre-commit tests * Append hook instead of refering with index Signed-off-by: Mahendra Paipuri --- .../test_pre_commit_5_reformat_markdown.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/external/pre_commit/test_pre_commit_5_reformat_markdown.py b/tests/external/pre_commit/test_pre_commit_5_reformat_markdown.py index d52c4865a..197cadcca 100644 --- a/tests/external/pre_commit/test_pre_commit_5_reformat_markdown.py +++ b/tests/external/pre_commit/test_pre_commit_5_reformat_markdown.py @@ -42,9 +42,9 @@ def test_pre_commit_hook_sync_reformat_code_and_markdown( jupytext_pre_commit_config["repos"][0]["hooks"][0]["additional_dependencies"] = [ "nbconvert==5.0.8" ] - jupytext_pre_commit_config["repos"][0]["hooks"][1] = jupytext_pre_commit_config[ - "repos" - ][0]["hooks"][0] + jupytext_pre_commit_config["repos"][0]["hooks"].append( + jupytext_pre_commit_config["repos"][0]["hooks"][0] + ) jupytext_pre_commit_config["repos"][0]["hooks"][1]["args"] = [ "--sync", "--pipe", @@ -58,11 +58,13 @@ def test_pre_commit_hook_sync_reformat_code_and_markdown( # Use python as language as we will need to install additional dependencies jupytext_pre_commit_config["repos"][0]["hooks"][0]["language"] = "python" jupytext_pre_commit_config["repos"][0]["hooks"][1]["language"] = "python" - jupytext_pre_commit_config["repos"][1] = { - "repo": "https://github.com/psf/black", - "rev": "22.3.0", - "hooks": [{"id": "black"}], - } + jupytext_pre_commit_config["repos"].append( + { + "repo": "https://github.com/psf/black", + "rev": "22.3.0", + "hooks": [{"id": "black"}], + } + ) with open(os.path.join(tmpdir, ".pre-commit-config.yaml"), "w") as file: yaml.dump(jupytext_pre_commit_config, file)