diff --git a/.github/workflows/test_deploy.yml b/.github/workflows/test_deploy.yml index 8457904f10..1a87b388d9 100644 --- a/.github/workflows/test_deploy.yml +++ b/.github/workflows/test_deploy.yml @@ -14,7 +14,8 @@ on: branches: - "**" - "!master" - +env: + RENKU_TEST_RECREATE_CACHE: "${{ (endsWith(github.ref, 'master') || endsWith(github.ref, 'develop') || startsWith(github.ref, 'refs/tags/') || startsWith(github.ref, 'refs/heads/release/' ) ) && '1' || '0' }}" jobs: set-matrix: runs-on: ubuntu-latest diff --git a/conftest.py b/conftest.py index a4c7e2bf64..715404df98 100644 --- a/conftest.py +++ b/conftest.py @@ -78,6 +78,7 @@ def pytest_configure(config): os.environ["RENKU_DISABLE_VERSION_CHECK"] = "1" # NOTE: Set an env var during during tests to mark that Renku is running in a test session. os.environ["RENKU_RUNNING_UNDER_TEST"] = "1" + os.environ["RENKU_SKIP_HOOK_CHECKS"] = "1" def pytest_unconfigure(config): @@ -85,3 +86,4 @@ def pytest_unconfigure(config): os.environ.pop("RENKU_SKIP_MIN_VERSION_CHECK", None) os.environ.pop("RENKU_DISABLE_VERSION_CHECK", None) os.environ.pop("RENKU_RUNNING_UNDER_TEST", None) + os.environ.pop("RENKU_SKIP_HOOK_CHECKS", None) diff --git a/renku/data/pre-commit.sh b/renku/data/pre-commit.sh index e413376cda..277843bef1 100755 --- a/renku/data/pre-commit.sh +++ b/renku/data/pre-commit.sh @@ -19,6 +19,10 @@ # RENKU HOOK. DO NOT REMOVE OR MODIFY. ###################################### +if [ "$RENKU_SKIP_HOOK_CHECKS" == "1" ]; then + exit 0 +fi + # Find all modified or added files, and do nothing if there aren't any. export RENKU_DISABLE_VERSION_CHECK=true diff --git a/tests/cli/fixtures/cli_projects.py b/tests/cli/fixtures/cli_projects.py index e65bd4a4f7..5559664e72 100644 --- a/tests/cli/fixtures/cli_projects.py +++ b/tests/cli/fixtures/cli_projects.py @@ -24,7 +24,9 @@ from renku.core.config import set_value from renku.infrastructure.repository import Repository +from renku.ui.cli import cli from tests.fixtures.repository import RenkuProject +from tests.utils import format_result_exception @pytest.fixture() @@ -131,3 +133,99 @@ def workflow_file_project(project, request) -> Generator[RenkuWorkflowFileProjec (project.path / "data" / "collection" / "colors.csv").write_text("\n".join(f"color-{i}" for i in range(99))) yield workflow_file_project + + +@pytest.fixture +def project_with_merge_conflict(runner, project, directory_tree, run_shell, cache_test_project): + """Project with a merge conflict.""" + if not cache_test_project.setup(): + result = runner.invoke(cli, ["mergetool", "install"]) + + assert 0 == result.exit_code, format_result_exception(result) + + # create a common dataset + result = runner.invoke( + cli, ["dataset", "add", "--copy", "--create", "shared-dataset", str(directory_tree)], catch_exceptions=False + ) + assert 0 == result.exit_code, format_result_exception(result) + + # Create a common workflow + output = run_shell('renku run --name "shared-workflow" echo "a unique string" > my_output_file') + + assert b"" == output[0] + assert output[1] is None + + # switch to a new branch + output = run_shell("git checkout -b remote-branch") + + assert b"Switched to a new branch 'remote-branch'\n" == output[0] + assert output[1] is None + + # edit the dataset + result = runner.invoke(cli, ["dataset", "edit", "-d", "remote description", "shared-dataset"]) + assert 0 == result.exit_code, format_result_exception(result) + + result = runner.invoke( + cli, ["dataset", "add", "--copy", "--create", "remote-dataset", str(directory_tree)], catch_exceptions=False + ) + assert 0 == result.exit_code, format_result_exception(result) + + # Create a new workflow + output = run_shell('renku run --name "remote-workflow" echo "a unique string" > remote_output_file') + + assert b"" == output[0] + assert output[1] is None + + # Create a downstream workflow + output = run_shell('renku run --name "remote-downstream-workflow" cp my_output_file my_remote_downstream') + + assert b"" == output[0] + assert output[1] is None + + # Create another downstream workflow + output = run_shell('renku run --name "remote-downstream-workflow2" cp remote_output_file my_remote_downstream2') + + assert b"" == output[0] + assert output[1] is None + + # Edit the project metadata + result = runner.invoke(cli, ["project", "edit", "-k", "remote"]) + + assert 0 == result.exit_code, format_result_exception(result) + + # Switch back to master + output = run_shell("git checkout master") + + assert b"Switched to branch 'master'\n" == output[0] + assert output[1] is None + + # Add a new dataset + result = runner.invoke( + cli, ["dataset", "add", "--copy", "--create", "local-dataset", str(directory_tree)], catch_exceptions=False + ) + assert 0 == result.exit_code, format_result_exception(result) + + # Create a local workflow + output = run_shell('renku run --name "local-workflow" echo "a unique string" > local_output_file') + + assert b"" == output[0] + assert output[1] is None + + # Create a local downstream workflow + output = run_shell('renku run --name "local-downstream-workflow" cp my_output_file my_local_downstream') + + assert b"" == output[0] + assert output[1] is None + + # Create another local downstream workflow + output = run_shell('renku run --name "local-downstream-workflow2" cp local_output_file my_local_downstream2') + + assert b"" == output[0] + assert output[1] is None + + # Edit the project in master as well + result = runner.invoke(cli, ["project", "edit", "-k", "local"]) + + assert 0 == result.exit_code, format_result_exception(result) + cache_test_project.save() + yield project diff --git a/tests/cli/fixtures/cli_workflow.py b/tests/cli/fixtures/cli_workflow.py index dd9b866a74..c959d2be26 100644 --- a/tests/cli/fixtures/cli_workflow.py +++ b/tests/cli/fixtures/cli_workflow.py @@ -17,21 +17,40 @@ import pytest +from renku.ui.cli import cli + @pytest.fixture -def workflow_graph(run_shell, project): +def workflow_graph(run_shell, project, cache_test_project): """Setup a project with a workflow graph.""" + cache_test_project.set_name("workflow_graph_fixture") + if not cache_test_project.setup(): + + def _run_workflow(name, command, extra_args=""): + output = run_shell(f"renku run --name {name} {extra_args} -- {command}") + # Assert not allocated stderr. + assert output[1] is None + + _run_workflow("r1", "echo 'test' > A") + _run_workflow("r2", "tee B C < A") + _run_workflow("r3", "cp A Z") + _run_workflow("r4", "cp B X") + _run_workflow("r5", "cat C Z > Y") + _run_workflow("r6", "bash -c 'cat X Y | tee R S'", extra_args="--input X --input Y --output R --output S") + _run_workflow("r7", "echo 'other' > H") + _run_workflow("r8", "tee I J < H") + cache_test_project.save() + + +@pytest.fixture +def project_with_dataset_and_workflows(runner, run_shell, project, directory_tree, cache_test_project): + """Project with a dataset and some workflows.""" + if not cache_test_project.setup(): + assert 0 == runner.invoke(cli, ["dataset", "create", "my-data"]).exit_code + assert 0 == runner.invoke(cli, ["dataset", "add", "--copy", "my-data", str(directory_tree)]).exit_code - def _run_workflow(name, command, extra_args=""): - output = run_shell(f"renku run --name {name} {extra_args} -- {command}") - # Assert not allocated stderr. - assert output[1] is None - - _run_workflow("r1", "echo 'test' > A") - _run_workflow("r2", "tee B C < A") - _run_workflow("r3", "cp A Z") - _run_workflow("r4", "cp B X") - _run_workflow("r5", "cat C Z > Y") - _run_workflow("r6", "bash -c 'cat X Y | tee R S'", extra_args="--input X --input Y --output R --output S") - _run_workflow("r7", "echo 'other' > H") - _run_workflow("r8", "tee I J < H") + assert run_shell('renku run --name run1 echo "my input string" > my_output_file')[1] is None + assert run_shell("renku run --name run2 cp my_output_file my_output_file2")[1] is None + assert run_shell("renku workflow compose my-composite-plan run1 run2")[1] is None + cache_test_project.save() + yield project diff --git a/tests/cli/test_datasets.py b/tests/cli/test_datasets.py index e873447cc9..f821fc40ea 100644 --- a/tests/cli/test_datasets.py +++ b/tests/cli/test_datasets.py @@ -1790,7 +1790,7 @@ def test_pull_data_from_lfs(runner, project, tmpdir, subdirectory, no_lfs_size_l assert 0 == result.exit_code, format_result_exception(result) -def test_lfs_hook(project_with_injection, subdirectory, large_file): +def test_lfs_hook(project_with_injection, subdirectory, large_file, enable_precommit_hook): """Test committing large files to Git.""" filenames = {"large-file", "large file with whitespace", "large*file?with wildcards"} @@ -1819,7 +1819,7 @@ def test_lfs_hook(project_with_injection, subdirectory, large_file): @pytest.mark.parametrize("use_env_var", [False, True]) -def test_lfs_hook_autocommit(runner, project, subdirectory, large_file, use_env_var): +def test_lfs_hook_autocommit(runner, project, subdirectory, large_file, use_env_var, enable_precommit_hook): """Test committing large files to Git gets automatically added to lfs.""" if use_env_var: os.environ["AUTOCOMMIT_LFS"] = "true" @@ -1851,7 +1851,7 @@ def test_lfs_hook_autocommit(runner, project, subdirectory, large_file, use_env_ assert filenames == tracked_lfs_files -def test_lfs_hook_can_be_avoided(runner, project, subdirectory, large_file): +def test_lfs_hook_can_be_avoided(runner, project, subdirectory, large_file, enable_precommit_hook): """Test committing large files to Git.""" result = runner.invoke( cli, ["--no-external-storage", "dataset", "add", "--copy", "-c", "my-dataset", str(large_file)] @@ -1860,7 +1860,7 @@ def test_lfs_hook_can_be_avoided(runner, project, subdirectory, large_file): assert "OK" in result.output -def test_datadir_hook(runner, project, subdirectory): +def test_datadir_hook(runner, project, subdirectory, enable_precommit_hook): """Test pre-commit hook fir checking datadir files.""" set_value(section="renku", key="check_datadir_files", value="true", global_only=True) diff --git a/tests/cli/test_graph.py b/tests/cli/test_graph.py index 45e5560f38..cca2e07e5c 100644 --- a/tests/cli/test_graph.py +++ b/tests/cli/test_graph.py @@ -15,26 +15,16 @@ # limitations under the License. """Test ``graph`` command.""" -import os - import pytest -from renku.core.constant import DEFAULT_DATA_DIR as DATA_DIR from renku.domain_model.dataset import Url from renku.ui.cli import cli from tests.utils import format_result_exception, modified_environ, with_dataset @pytest.mark.parametrize("revision", ["", "HEAD", "HEAD^", "HEAD^..HEAD"]) -def test_graph_export_validation(runner, project, directory_tree, run, revision): +def test_graph_export_validation(runner, project_with_dataset_and_workflows, revision): """Test graph validation when exporting.""" - assert 0 == runner.invoke(cli, ["dataset", "add", "--copy", "-c", "my-data", str(directory_tree)]).exit_code - - file1 = project.path / DATA_DIR / "my-data" / directory_tree.name / "file1" - file2 = project.path / DATA_DIR / "my-data" / directory_tree.name / "dir1" / "file2" - assert 0 == run(["run", "head", str(file1)], stdout="out1") - assert 0 == run(["run", "tail", str(file2)], stdout="out2") - result = runner.invoke(cli, ["graph", "export", "--format", "json-ld", "--strict", "--revision", revision]) assert 0 == result.exit_code, format_result_exception(result) @@ -51,17 +41,17 @@ def test_graph_export_validation(runner, project, directory_tree, run, revision) assert "https://renkulab.io" in result.output # Make sure that nothing has changed during export which is a read-only operation - assert not project.repository.is_dirty() + assert not project_with_dataset_and_workflows.repository.is_dirty() @pytest.mark.serial @pytest.mark.shelled -def test_graph_export_strict_run(runner, project, run_shell): - """Test graph export output of run command.""" - # Run a shell command with pipe. - assert run_shell('renku run --name run1 echo "my input string" > my_output_file')[1] is None - assert run_shell("renku run --name run2 cp my_output_file my_output_file2")[1] is None - assert run_shell("renku workflow compose my-composite-plan run1 run2")[1] is None +def test_graph_export_strict( + runner, + project_with_dataset_and_workflows, + run_shell, +): + """Test strict graph export output command.""" # Assert created output file. result = runner.invoke(cli, ["graph", "export", "--full", "--strict", "--format=json-ld"]) @@ -71,55 +61,32 @@ def test_graph_export_strict_run(runner, project, run_shell): assert "my_output_file2" in result.output assert "my-composite-plan" in result.output - assert run_shell("renku workflow remove composite")[1] is None - assert run_shell("renku workflow remove run2")[1] is None - - # Assert created output file. - result = runner.invoke(cli, ["graph", "export", "--strict", "--format=json-ld"]) - assert 0 == result.exit_code, format_result_exception(result) - - -def test_graph_export_strict_dataset(tmpdir, runner, project, subdirectory): - """Test output of graph export for dataset add.""" - result = runner.invoke(cli, ["dataset", "create", "my-dataset"]) - assert 0 == result.exit_code, format_result_exception(result) - paths = [] - test_paths = [] - for i in range(3): - new_file = tmpdir.join(f"file_{i}") - new_file.write(str(i)) - paths.append(str(new_file)) - test_paths.append(os.path.relpath(str(new_file), str(project.path))) - - # add data - result = runner.invoke(cli, ["dataset", "add", "--copy", "my-dataset"] + paths) + result = runner.invoke(cli, ["graph", "export", "--strict", "--format=json-ld", "--revision", "HEAD^^^^"]) assert 0 == result.exit_code, format_result_exception(result) - result = runner.invoke(cli, ["graph", "export", "--strict", "--format=json-ld", "--revision", "HEAD"]) - assert 0 == result.exit_code, format_result_exception(result) - assert all(p in result.output for p in test_paths), result.output - # check that only most recent dataset is exported assert 1 == result.output.count("http://schema.org/Dataset") - # NOTE: Don't pass ``--full`` to check it's the default action. + assert run_shell("renku workflow remove composite")[1] is None + assert run_shell("renku workflow remove run2")[1] is None + + # Assert created output file. result = runner.invoke(cli, ["graph", "export", "--strict", "--format=json-ld"]) assert 0 == result.exit_code, format_result_exception(result) - assert all(p in result.output for p in test_paths), result.output - # check that all datasets are exported + assert all(p in result.output for p in ["my_output_file2", "my_output_file"]), result.output + assert 2 == result.output.count("http://schema.org/Dataset") # remove and readd dataset - result = runner.invoke(cli, ["dataset", "rm", "my-dataset"]) + result = runner.invoke(cli, ["dataset", "rm", "my-data"]) assert 0 == result.exit_code, format_result_exception(result) - result = runner.invoke(cli, ["dataset", "create", "my-dataset"]) + result = runner.invoke(cli, ["dataset", "create", "my-data"]) assert 0 == result.exit_code, format_result_exception(result) result = runner.invoke(cli, ["graph", "export", "--strict", "--format=json-ld"]) assert 0 == result.exit_code, format_result_exception(result) - assert all(p in result.output for p in test_paths), result.output # check that all datasets are exported assert 4 == result.output.count("http://schema.org/Dataset") diff --git a/tests/cli/test_merge.py b/tests/cli/test_merge.py index a0af7c28f5..c5546c5e7e 100644 --- a/tests/cli/test_merge.py +++ b/tests/cli/test_merge.py @@ -23,96 +23,8 @@ from tests.utils import format_result_exception -def test_mergetool(runner, project, directory_tree, run_shell, with_injection): +def test_mergetool(runner, project_with_merge_conflict, directory_tree, run_shell, with_injection): """Test that merge tool can merge renku metadata.""" - result = runner.invoke(cli, ["mergetool", "install"]) - - assert 0 == result.exit_code, format_result_exception(result) - - # create a common dataset - result = runner.invoke( - cli, ["dataset", "add", "--copy", "--create", "shared-dataset", str(directory_tree)], catch_exceptions=False - ) - assert 0 == result.exit_code, format_result_exception(result) - - # Create a common workflow - output = run_shell('renku run --name "shared-workflow" echo "a unique string" > my_output_file') - - assert b"" == output[0] - assert output[1] is None - - # switch to a new branch - output = run_shell("git checkout -b remote-branch") - - assert b"Switched to a new branch 'remote-branch'\n" == output[0] - assert output[1] is None - - # edit the dataset - result = runner.invoke(cli, ["dataset", "edit", "-d", "remote description", "shared-dataset"]) - assert 0 == result.exit_code, format_result_exception(result) - - result = runner.invoke( - cli, ["dataset", "add", "--copy", "--create", "remote-dataset", str(directory_tree)], catch_exceptions=False - ) - assert 0 == result.exit_code, format_result_exception(result) - - # Create a new workflow - output = run_shell('renku run --name "remote-workflow" echo "a unique string" > remote_output_file') - - assert b"" == output[0] - assert output[1] is None - - # Create a downstream workflow - output = run_shell('renku run --name "remote-downstream-workflow" cp my_output_file my_remote_downstream') - - assert b"" == output[0] - assert output[1] is None - - # Create another downstream workflow - output = run_shell('renku run --name "remote-downstream-workflow2" cp remote_output_file my_remote_downstream2') - - assert b"" == output[0] - assert output[1] is None - - # Edit the project metadata - result = runner.invoke(cli, ["project", "edit", "-k", "remote"]) - - assert 0 == result.exit_code, format_result_exception(result) - - # Switch back to master - output = run_shell("git checkout master") - - assert b"Switched to branch 'master'\n" == output[0] - assert output[1] is None - - # Add a new dataset - result = runner.invoke( - cli, ["dataset", "add", "--copy", "--create", "local-dataset", str(directory_tree)], catch_exceptions=False - ) - assert 0 == result.exit_code, format_result_exception(result) - - # Create a local workflow - output = run_shell('renku run --name "local-workflow" echo "a unique string" > local_output_file') - - assert b"" == output[0] - assert output[1] is None - - # Create a local downstream workflow - output = run_shell('renku run --name "local-downstream-workflow" cp my_output_file my_local_downstream') - - assert b"" == output[0] - assert output[1] is None - - # Create another local downstream workflow - output = run_shell('renku run --name "local-downstream-workflow2" cp local_output_file my_local_downstream2') - - assert b"" == output[0] - assert output[1] is None - - # Edit the project in master as well - result = runner.invoke(cli, ["project", "edit", "-k", "local"]) - - assert 0 == result.exit_code, format_result_exception(result) # Merge branches output = run_shell("git merge --no-edit remote-branch") diff --git a/tests/cli/test_remove.py b/tests/cli/test_remove.py index cfac23414c..11c349d484 100644 --- a/tests/cli/test_remove.py +++ b/tests/cli/test_remove.py @@ -36,12 +36,12 @@ def test_remove_dataset_file(isolated_runner, project, tmpdir, subdirectory, dat result = isolated_runner.invoke(cli, ["dataset", "add", "--copy", "testing", source.strpath]) assert 0 == result.exit_code, format_result_exception(result) - path = project.path / datadir / "remove_dataset.file" - assert path.exists() - result = isolated_runner.invoke(cli, ["doctor"]) assert 0 == result.exit_code, format_result_exception(result) + path = project.path / datadir / "remove_dataset.file" + assert path.exists() + result = isolated_runner.invoke(cli, ["rm", str(project.path / datadir)]) assert 0 == result.exit_code, format_result_exception(result) diff --git a/tests/cli/test_rollback.py b/tests/cli/test_rollback.py index 798eddf54d..cf0923292c 100644 --- a/tests/cli/test_rollback.py +++ b/tests/cli/test_rollback.py @@ -19,7 +19,7 @@ from tests.utils import format_result_exception -def test_rollback(runner, project): +def test_rollback(runner, project, cache_test_project): """Test renku rollback.""" result = runner.invoke(cli, ["run", "--name", "run1", "touch", "foo"]) assert 0 == result.exit_code, format_result_exception(result) diff --git a/tests/cli/test_template.py b/tests/cli/test_template.py index 89bbbe0c91..e7d1f4e885 100644 --- a/tests/cli/test_template.py +++ b/tests/cli/test_template.py @@ -305,7 +305,7 @@ def test_template_update_dry_run(runner, project): assert commit_sha_before == project.repository.head.commit.hexsha -def test_git_hook_for_modified_immutable_template_files(runner, project_with_template): +def test_git_hook_for_modified_immutable_template_files(runner, project_with_template, enable_precommit_hook): """Test check for modified immutable template files.""" (project_with_template.path / "immutable.file").write_text("Locally modified immutable files") diff --git a/tests/cli/test_update.py b/tests/cli/test_update.py index 7cd6950d3f..cb7046ea99 100644 --- a/tests/cli/test_update.py +++ b/tests/cli/test_update.py @@ -484,7 +484,15 @@ def test_update_with_execute(runner, project, renku_cli, provider): assert ( 0 == renku_cli( - "workflow", "execute", "-p", provider, "--set", f"input-2={source2}", "--set", f"output-3={output2}", "test" + "workflow", + "execute", + "-p", + provider, + "--set", + f"input-2={source2}", + "--set", + f"output-3={output2}", + "test", ).exit_code ) diff --git a/tests/cli/test_workflow.py b/tests/cli/test_workflow.py index 19d970e5e0..b5ee449e83 100644 --- a/tests/cli/test_workflow.py +++ b/tests/cli/test_workflow.py @@ -616,10 +616,18 @@ def test_workflow_show_outputs_with_directory(runner, project, run): ], ) def test_workflow_execute_command( - runner, run_shell, project, capsys, with_injection, provider, yaml, skip_metadata_update, workflows, parameters + runner, + run_shell, + project, + capsys, + with_injection, + provider, + yaml, + skip_metadata_update, + workflows, + parameters, ): """Test workflow execute.""" - for wf in workflows: output = run_shell(f"renku run --name {wf[0]} -- {wf[1]}") # Assert expected empty stdout. @@ -1233,15 +1241,13 @@ def test_workflow_iterate_command_with_parameter_set(runner, run_shell, project, assert "2.0\n" == output.read_text() - result = run_shell(f"renku workflow iterate -p {provider} --map parameter-2=[0.1,0.3,0.5,0.8,0.95] run1") + result = run_shell(f"renku workflow iterate -p {provider} --map parameter-2=[0.1,0.3,0.95] run1") # Assert not allocated stderr. assert result[1] is None assert output.read_text() in [ "0.1\n", "0.3\n", - "0.5\n", - "0.8\n", "0.95\n", ] diff --git a/tests/core/fixtures/core_datasets.py b/tests/core/fixtures/core_datasets.py index af4f1c586d..3edfa9c805 100644 --- a/tests/core/fixtures/core_datasets.py +++ b/tests/core/fixtures/core_datasets.py @@ -1,5 +1,6 @@ -# Copyright Swiss Data Science Center (SDSC). A partnership between -# École Polytechnique Fédérale de Lausanne (EPFL) and +# +# Copyright 2021 Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and # Eidgenössische Technische Hochschule Zürich (ETHZ). # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -45,21 +46,28 @@ def request_callback(request): @pytest.fixture -def project_with_datasets(project, directory_tree, with_injection) -> Generator[RenkuProject, None, None]: +def project_with_datasets( + project, directory_tree, with_injection, cache_test_project +) -> Generator[RenkuProject, None, None]: """A project with datasets.""" from renku.domain_model.provenance.agent import Person - person_1 = Person.from_string("P1 [IANA]") - person_2 = Person.from_string("P2 ") + cache_test_project.set_name("project_with_datasets_fixture") + if not cache_test_project.setup(): + person_1 = Person.from_string("P1 [IANA]") + person_2 = Person.from_string("P2 ") - with with_injection(): - create_dataset(slug="dataset-1", keywords=["dataset", "1"], creators=[person_1]) + with with_injection(): + create_dataset(slug="dataset-1", keywords=["dataset", "1"], creators=[person_1]) - dataset = add_to_dataset("dataset-2", urls=[str(p) for p in directory_tree.glob("*")], create=True, copy=True) - dataset.keywords = ["dataset", "2"] - dataset.creators = [person_1, person_2] + dataset = add_to_dataset( + "dataset-2", urls=[str(p) for p in directory_tree.glob("*")], create=True, copy=True + ) + dataset.keywords = ["dataset", "2"] + dataset.creators = [person_1, person_2] - project.repository.add(all=True) - project.repository.commit("add files to datasets") + project.repository.add(all=True) + project.repository.commit("add files to datasets") + cache_test_project.save() yield project diff --git a/tests/core/fixtures/core_workflow.py b/tests/core/fixtures/core_workflow.py index dda9195b52..bffd9042fd 100644 --- a/tests/core/fixtures/core_workflow.py +++ b/tests/core/fixtures/core_workflow.py @@ -90,7 +90,9 @@ def create_run(name: str) -> Plan: @pytest.fixture -def project_with_runs(project_with_creation_date, with_injection) -> Generator[RenkuProject, None, None]: +def project_with_runs( + project_with_creation_date, with_injection, cache_test_project +) -> Generator[RenkuProject, None, None]: """A project with runs.""" from renku.domain_model.provenance.activity import Activity from renku.infrastructure.gateway.activity_gateway import ActivityGateway @@ -105,44 +107,47 @@ def create_activity(plan, date, index) -> Activity: ended_at_time=date + timedelta(seconds=1), ) - date_1 = datetime(2022, 5, 20, 0, 42, 0, tzinfo=timezone.utc) - date_2 = datetime(2022, 5, 20, 0, 43, 0, tzinfo=timezone.utc) - - plan_1 = create_dummy_plan( - command="command-1", - date_created=date_1, - description="First plan", - index=1, - inputs=["input"], - keywords=["plans", "1"], - name="plan-1", - outputs=[("intermediate", "stdout")], - parameters=[("parameter-1", "42", "-n ")], - success_codes=[0, 1], - ) - - plan_2 = create_dummy_plan( - command="command-2", - date_created=date_2, - description="Second plan", - index=2, - inputs=["intermediate"], - keywords=["plans", "2"], - name="plan-2", - outputs=[("output", "stdout")], - parameters=[("int-parameter", 43, "-n "), ("str-parameter", "some value", None)], - ) - - with with_injection(): - activity_1 = create_activity(plan_1, date_1, index=1) - activity_2 = create_activity(plan_2, date_2, index=2) - - activity_gateway = ActivityGateway() - - activity_gateway.add(activity_1) - activity_gateway.add(activity_2) - - project_with_creation_date.repository.add(all=True) - project_with_creation_date.repository.commit("Add runs") + cache_test_project.set_name("project_with_runs_fixture") + if not cache_test_project.setup(): + date_1 = datetime(2022, 5, 20, 0, 42, 0, tzinfo=timezone.utc) + date_2 = datetime(2022, 5, 20, 0, 43, 0, tzinfo=timezone.utc) + + plan_1 = create_dummy_plan( + command="command-1", + date_created=date_1, + description="First plan", + index=1, + inputs=["input"], + keywords=["plans", "1"], + name="plan-1", + outputs=[("intermediate", "stdout")], + parameters=[("parameter-1", "42", "-n ")], + success_codes=[0, 1], + ) + + plan_2 = create_dummy_plan( + command="command-2", + date_created=date_2, + description="Second plan", + index=2, + inputs=["intermediate"], + keywords=["plans", "2"], + name="plan-2", + outputs=[("output", "stdout")], + parameters=[("int-parameter", 43, "-n "), ("str-parameter", "some value", None)], + ) + + with with_injection(): + activity_1 = create_activity(plan_1, date_1, index=1) + activity_2 = create_activity(plan_2, date_2, index=2) + + activity_gateway = ActivityGateway() + + activity_gateway.add(activity_1) + activity_gateway.add(activity_2) + + project_with_creation_date.repository.add(all=True) + project_with_creation_date.repository.commit("Add runs") + cache_test_project.save() yield project_with_creation_date diff --git a/tests/data/repo-cache/project_with_datasets_fixture.tar.gz b/tests/data/repo-cache/project_with_datasets_fixture.tar.gz new file mode 100644 index 0000000000..ec9e95f522 Binary files /dev/null and b/tests/data/repo-cache/project_with_datasets_fixture.tar.gz differ diff --git a/tests/data/repo-cache/project_with_runs_fixture.tar.gz b/tests/data/repo-cache/project_with_runs_fixture.tar.gz new file mode 100644 index 0000000000..b07b0b4ed5 Binary files /dev/null and b/tests/data/repo-cache/project_with_runs_fixture.tar.gz differ diff --git a/tests/data/repo-cache/test_graph_export_strict.tar.gz b/tests/data/repo-cache/test_graph_export_strict.tar.gz new file mode 100644 index 0000000000..b6917912ed Binary files /dev/null and b/tests/data/repo-cache/test_graph_export_strict.tar.gz differ diff --git a/tests/data/repo-cache/test_graph_export_validation_HEAD_..HEAD_.tar.gz b/tests/data/repo-cache/test_graph_export_validation_HEAD_..HEAD_.tar.gz new file mode 100644 index 0000000000..9a2bfd6f27 Binary files /dev/null and b/tests/data/repo-cache/test_graph_export_validation_HEAD_..HEAD_.tar.gz differ diff --git a/tests/data/repo-cache/test_graph_export_validation_HEAD_.tar.gz b/tests/data/repo-cache/test_graph_export_validation_HEAD_.tar.gz new file mode 100644 index 0000000000..199afae821 Binary files /dev/null and b/tests/data/repo-cache/test_graph_export_validation_HEAD_.tar.gz differ diff --git a/tests/data/repo-cache/test_graph_export_validation_HEAD__.tar.gz b/tests/data/repo-cache/test_graph_export_validation_HEAD__.tar.gz new file mode 100644 index 0000000000..40f3b57f8f Binary files /dev/null and b/tests/data/repo-cache/test_graph_export_validation_HEAD__.tar.gz differ diff --git a/tests/data/repo-cache/test_graph_export_validation__.tar.gz b/tests/data/repo-cache/test_graph_export_validation__.tar.gz new file mode 100644 index 0000000000..62a969162e Binary files /dev/null and b/tests/data/repo-cache/test_graph_export_validation__.tar.gz differ diff --git a/tests/data/repo-cache/test_mergetool.tar.gz b/tests/data/repo-cache/test_mergetool.tar.gz new file mode 100644 index 0000000000..8639b51339 Binary files /dev/null and b/tests/data/repo-cache/test_mergetool.tar.gz differ diff --git a/tests/data/repo-cache/workflow_graph_fixture.tar.gz b/tests/data/repo-cache/workflow_graph_fixture.tar.gz new file mode 100644 index 0000000000..10939d574f Binary files /dev/null and b/tests/data/repo-cache/workflow_graph_fixture.tar.gz differ diff --git a/tests/fixtures/common.py b/tests/fixtures/common.py index 6a8c77f854..c597329eb8 100644 --- a/tests/fixtures/common.py +++ b/tests/fixtures/common.py @@ -16,6 +16,7 @@ """Renku common fixtures.""" import os +import shutil from pathlib import Path from typing import Generator, List @@ -81,9 +82,70 @@ def large_file(tmp_path): yield path +@pytest.fixture +def enable_precommit_hook(): + """Enable running precommit hooks for the test.""" + os.environ["RENKU_SKIP_HOOK_CHECKS"] = "0" + yield + os.environ["RENKU_SKIP_HOOK_CHECKS"] = "1" + + @pytest.fixture def transaction_id(project) -> Generator[str, None, None]: """Return current transaction ID.""" from renku.domain_model.project_context import project_context yield project_context.transaction_id + + +@pytest.fixture +def cache_test_project(request, project): + """Caches a renku project repository for reuse between tests.""" + marker = request.node.get_closest_marker("project_cache_name") + + if marker: + cache_name = marker.args[0] + else: + cache_name = "".join(x if x.isalnum() or x in "-_." else "_" for x in request.node.name) + + class _ProjectRepoCache: + def __init__(self, cache_name: str) -> None: + self.set_name(cache_name) + + def set_name(self, cache_name: str) -> None: + """Change the name of the cache.""" + self.cache_name = cache_name + self.cache_dir = Path(request.config.rootdir) / Path( + os.environ.get("RENKU_TEST_PROJECT_CACHE_DIR", "tests/data/repo-cache") + ) + self.cache_dir.mkdir(exist_ok=True) + self.filename = self.cache_dir / f"{self.cache_name}.tar.gz" + + def delete_project_contents(self) -> None: + """Delete the contents of the project directory.""" + subdir_paths = [str(project.path / p) for p in ["some", "some/sub", "some/sub/directory"]] + for root, dirs, files in os.walk(project.path): + for f in files: + os.unlink(os.path.join(root, f)) + for d in dirs: + path = os.path.join(root, d) + # NOTE: Don't delete `subdirectory` fixture dir + if path not in subdir_paths: + shutil.rmtree(path) + + def save(self): + """Save state of the project directory.""" + self.filename.unlink(missing_ok=True) + shutil.make_archive(str(self.cache_dir / self.cache_name), "gztar", project.path) + + def setup(self) -> bool: + """Recreate state of project directory from previous test, if applicable.""" + if not self.filename.exists() or os.environ.get("RENKU_TEST_RECREATE_CACHE", "0") == "1": + return False + + self.delete_project_contents() + shutil.unpack_archive(self.filename, project.path, "gztar") + return True + + assert cache_name is not None + return _ProjectRepoCache(cache_name)