diff --git a/.appveyor.yml b/.appveyor.yml new file mode 100644 index 0000000..afa8088 --- /dev/null +++ b/.appveyor.yml @@ -0,0 +1,19 @@ +environment: + + matrix: + - PYTHON: "C:\\Python36" +# - PYTHON: "C:\\Python37" + +build: off + +install: + - "%PYTHON%\\python.exe -m pip install -r requirements.txt" + - "%PYTHON%\\python.exe setup.py install" + +test_script: + - "%PYTHON%\\python.exe setup.py test" + +branches: + only: + - master + - /release-v[0-9]+/ diff --git a/arca/_arca.py b/arca/_arca.py index 8c2ced1..847539c 100644 --- a/arca/_arca.py +++ b/arca/_arca.py @@ -1,5 +1,6 @@ import hashlib import json +import platform import re from collections import defaultdict from datetime import datetime @@ -139,7 +140,22 @@ def validate_repo_url(self, repo: str): :raise ValueError: If the URL is not valid """ # that should match valid git repos - if not isinstance(repo, str) or not re.match(r"^(https?|file)://[\w._\-/~]*[.git]?/?$", repo): + valid = True + + url_regex = re.compile(r"^https?://[\w._\-/~]*[.git]?/?$") + + if not isinstance(repo, str): + valid = False + else: + if platform.system() == "Windows": + if not url_regex.match(repo) and not re.match(r"file://(localhost)?/[a-zA-Z]:\\[\\\S|*\S]?.*", repo): + valid = False + + else: + if not url_regex.match(repo) and not re.match(r"file://[\w._\-/~]*[.git]?/?$", repo): + valid = False + + if not valid: raise ValueError(f"{repo} is not a valid http[s] or file:// git repository.") def repo_id(self, repo: str) -> str: diff --git a/requirements.txt b/requirements.txt index 5b7a6b9..e3ee38c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ gitpython pytest dogpile.cache==0.6.5 -docker~=3.1.0 +docker~=3.4.1 pytest-flake8 pytest-cov pytest-mock diff --git a/setup.py b/setup.py index 2da721a..f0563b9 100644 --- a/setup.py +++ b/setup.py @@ -39,10 +39,10 @@ def long_description(): ], extras_require={ "docker": [ - "docker~=3.2.1", + "docker~=3.4.1", ], "vagrant": [ - "docker~=3.2.1", + "docker~=3.4.1", "python-vagrant", "fabric3", ], diff --git a/tests/common.py b/tests/common.py index 9926962..c980e7c 100644 --- a/tests/common.py +++ b/tests/common.py @@ -1,8 +1,10 @@ import os - +from pathlib import Path if os.environ.get("TRAVIS", False): BASE_DIR = "/home/travis/build/{}/test_loc".format(os.environ.get("TRAVIS_REPO_SLUG", "mikicz/arca")) +elif os.environ.get("APPVEYOR", False): + BASE_DIR = str(Path(os.environ["APPVEYOR_BUILD_FOLDER"]) / "test_loc") else: BASE_DIR = "/tmp/arca/test" diff --git a/tests/conftest.py b/tests/conftest.py index d8757db..a7067d1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,3 +1,4 @@ +import platform import shutil import tempfile from pathlib import Path @@ -16,7 +17,12 @@ def create_temp_repo(file) -> TempRepo: git_dir = Path(tempfile.mkdtemp()) repo = Repo.init(str(git_dir)) - return TempRepo(repo, git_dir, f"file://{git_dir}", "master", git_dir / file) + file_url = f"file://{git_dir}" + + if platform.system() == "Windows": + file_url = f"file:///{git_dir}" + + return TempRepo(repo, git_dir, file_url, "master", git_dir / file) @pytest.fixture() @@ -30,6 +36,7 @@ def temp_repo_func(): yield temp_repo + temp_repo.repo.close() shutil.rmtree(str(temp_repo.repo_path)) @@ -44,4 +51,5 @@ def temp_repo_static(): yield temp_repo + temp_repo.repo.close() shutil.rmtree(str(temp_repo.repo_path)) diff --git a/tests/test_arca_class.py b/tests/test_arca_class.py index c7abde1..29e766a 100644 --- a/tests/test_arca_class.py +++ b/tests/test_arca_class.py @@ -1,4 +1,5 @@ # encoding=utf-8 +import platform import re import shutil from pathlib import Path @@ -32,18 +33,33 @@ class NotASubclassClass: @pytest.mark.parametrize(["url", "valid"], [ + # http/s ("http://host.xz/path/to/repo.git/", True), ("https://host.xz/path/to/repo.git/", True), ("http://host.xz/path/to/repo.git", True), ("https://host.xz/path/to/repo.git", True), - ("file:///path/to/repo.git/", True), - ("file://~/path/to/repo.git/", True), ("http://host.xz/path/to/repo/", True), ("https://host.xz/path/to/repo/", True), ("http://host.xz/path/to/repo", True), ("https://host.xz/path/to/repo", True), - ("file:///path/to/repo.git", True), - ("file://~/path/to/repo.git", True), + + # linux paths + pytest.param("file:///path/to/repo.git/", True, + marks=pytest.mark.skipif(platform.system() == "Windows", reason="Linux Path")), + pytest.param("file://~/path/to/repo.git/", True, + marks=pytest.mark.skipif(platform.system() == "Windows", reason="Linux Path")), + pytest.param("file:///path/to/repo.git", True, + marks=pytest.mark.skipif(platform.system() == "Windows", reason="Linux Path")), + pytest.param("file://~/path/to/repo.git", True, + marks=pytest.mark.skipif(platform.system() == "Windows", reason="Linux Path")), + + # windows paths + pytest.param("file:///C:\\user\\path \\to\\repo", True, + marks=pytest.mark.skipif(platform.system() != "Windows", reason="Windows Path")), + pytest.param("file:///c:\\user\\path \\to\\repo", True, + marks=pytest.mark.skipif(platform.system() != "Windows", reason="Windows Path")), + + # ssh ("git://host.xz/path/to/repo.git/", False), ("git://host.xz/~user/path/to/repo.git/", False), ("ssh://host.xz/path/to/repo.git/", False), @@ -177,6 +193,7 @@ def test_depth(temp_repo_static): cloned_repo, cloned_repo_path = arca.get_files(temp_repo_static.url, temp_repo_static.branch) assert cloned_repo.commit().count() == 2 + cloned_repo.close() shutil.rmtree(str(cloned_repo_path)) # test that when setting a certain depth, at least the depth is pulled (in case of merges etc) @@ -194,6 +211,7 @@ def test_depth(temp_repo_static): cloned_repo, cloned_repo_path = arca.get_files(temp_repo_static.url, temp_repo_static.branch) assert cloned_repo.commit().count() == before_second_pull + 1 + cloned_repo.close() shutil.rmtree(str(cloned_repo_path)) # test when setting depth bigger than repo size, no fictional commits are included @@ -202,6 +220,7 @@ def test_depth(temp_repo_static): assert cloned_repo.commit().count() == 22 # 20 plus the 2 extra commits + cloned_repo.close() shutil.rmtree(str(cloned_repo_path)) # test no limit @@ -237,7 +256,7 @@ def test_reference(): arca = Arca(base_dir=BASE_DIR) branch = "master" - git_dir_1 = Path("/tmp/arca/") / str(uuid4()) + git_dir_1 = Path(BASE_DIR) / str(uuid4()) git_url_1 = f"file://{git_dir_1}" filepath_1 = git_dir_1 / "test_file.txt" repo_1 = Repo.init(git_dir_1) @@ -252,13 +271,15 @@ def test_reference(): # test nonexistent reference - cloned_repo, cloned_repo_path = arca.get_files(git_url_1, branch, reference=Path("/tmp/arca/") / str(uuid4())) + cloned_repo, cloned_repo_path = arca.get_files(git_url_1, branch, reference=Path(BASE_DIR) / str(uuid4())) assert (cloned_repo_path / "test_file.txt").read_text() == last_uuid + + cloned_repo.close() shutil.rmtree(str(cloned_repo_path)) # test existing reference with no common commits - git_dir_2 = Path("/tmp/arca/") / str(uuid4()) + git_dir_2 = Path(BASE_DIR) / str(uuid4()) filepath_2 = git_dir_2 / "test_file.txt" repo_2 = Repo.init(git_dir_2) @@ -269,11 +290,13 @@ def test_reference(): cloned_repo, cloned_repo_path = arca.get_files(git_url_1, branch, reference=git_dir_2) assert (cloned_repo_path / "test_file.txt").read_text() == last_uuid + + cloned_repo.close() shutil.rmtree(str(cloned_repo_path)) # test existing reference with common commits - git_dir_3 = Path("/tmp/arca/") / str(uuid4()) + git_dir_3 = Path(BASE_DIR) / str(uuid4()) git_url_3 = f"file://{git_dir_3}" filepath_3 = git_dir_3 / "test_file.txt" repo_3 = repo_1.clone(str(git_dir_3)) # must pass string, fails otherwise @@ -334,7 +357,7 @@ def test_get_reference_repository(temp_repo_static): def test_pull_error(): arca = Arca(base_dir=BASE_DIR) - git_dir = Path("/tmp/arca/") / str(uuid4()) + git_dir = Path(BASE_DIR) / str(uuid4()) git_url = f"file://{git_dir}" with pytest.raises(PullError): @@ -351,6 +374,8 @@ def test_pull_error(): with pytest.raises(PullError): arca.get_files(git_url, "some_branch") + repo.close() + shutil.rmtree(str(git_dir)) with pytest.raises(PullError): diff --git a/tests/test_backends.py b/tests/test_backends.py index fd507c9..fb318d6 100644 --- a/tests/test_backends.py +++ b/tests/test_backends.py @@ -56,7 +56,7 @@ def test_backends(temp_repo_func, backend, requirements_location, file_location) assert arca.run(temp_repo_func.url, temp_repo_func.branch, task).output == "Some string" - filepath.write_text(SECOND_RETURN_STR_FUNCTION) + filepath.write_text(SECOND_RETURN_STR_FUNCTION, encoding="utf-8") temp_repo_func.repo.create_head("new_branch") temp_repo_func.repo.create_tag("test_tag") temp_repo_func.repo.index.add([str(filepath)]) diff --git a/tests/test_runner.py b/tests/test_runner.py index 4cea1e7..c538ee8 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -1,4 +1,5 @@ import json +import os import tempfile from pathlib import Path @@ -35,7 +36,8 @@ "kwargs": [1, 2, 3]}, ]) def test_definition_corruption(definition): - _, file = tempfile.mkstemp() + fd, file = tempfile.mkstemp() + file = Path(file) if isinstance(definition, dict): @@ -49,6 +51,7 @@ def test_definition_corruption(definition): assert output["error"] assert output["reason"] == "corrupted_definition" + os.close(fd) file.unlink() @@ -59,7 +62,7 @@ def test_definition_corruption(definition): ("arca", "Arca.some_random_method"), ]) def test_import_error(module_name, object_name): - _, file = tempfile.mkstemp() + fd, file = tempfile.mkstemp() file = Path(file) file.write_text(json.dumps({ @@ -74,6 +77,7 @@ def test_import_error(module_name, object_name): assert output["error"] assert output["reason"] == "import" + os.close(fd) file.unlink() @@ -86,7 +90,7 @@ def test_run(mocker, func, result): load = mocker.patch("arca._runner.EntryPoint.load") load.return_value = func - _, file = tempfile.mkstemp() + fd, file = tempfile.mkstemp() file = Path(file) file.write_text(json.dumps({ @@ -104,6 +108,7 @@ def test_run(mocker, func, result): assert output["success"] is False assert result.__name__ in output["error"] + os.close(fd) file.unlink() @@ -121,7 +126,7 @@ def func(カ): load.return_value = func - _, file = tempfile.mkstemp() + fd, file = tempfile.mkstemp() file = Path(file) file.write_text(Task("library.mod:func", args=args, kwargs=kwargs).json) @@ -130,6 +135,7 @@ def func(カ): assert Result(output).output == result + os.close(fd) file.unlink() diff --git a/tests/test_single_pull.py b/tests/test_single_pull.py index 99f6341..1a6011a 100644 --- a/tests/test_single_pull.py +++ b/tests/test_single_pull.py @@ -13,7 +13,7 @@ def test_single_pull(temp_repo_func, mocker): assert arca.run(temp_repo_func.url, temp_repo_func.branch, task).output == "Some string" assert arca._pull.call_count == 1 - temp_repo_func.file_path.write_text(SECOND_RETURN_STR_FUNCTION) + temp_repo_func.file_path.write_text(SECOND_RETURN_STR_FUNCTION, encoding="utf-8") temp_repo_func.repo.index.add([str(temp_repo_func.file_path)]) temp_repo_func.repo.index.commit("Updated function") @@ -40,7 +40,7 @@ def test_pull_efficiency(temp_repo_func, mocker): assert arca.run(temp_repo_func.url, temp_repo_func.branch, task).output == "Some string" assert arca._pull.call_count == 2 - temp_repo_func.file_path.write_text(SECOND_RETURN_STR_FUNCTION) + temp_repo_func.file_path.write_text(SECOND_RETURN_STR_FUNCTION, encoding="utf-8") temp_repo_func.repo.index.add([str(temp_repo_func.file_path)]) temp_repo_func.repo.index.commit("Updated function") diff --git a/tests/test_vagrant.py b/tests/test_vagrant.py index 1d613ac..052d9de 100644 --- a/tests/test_vagrant.py +++ b/tests/test_vagrant.py @@ -11,8 +11,8 @@ def test_validation(): - if os.environ.get("TRAVIS", False): - pytest.skip("Vagrant doesn't work on Travis") + if os.environ.get("CI", False): + pytest.skip("Vagrant doesn't work on CIs") backend = VagrantBackend() @@ -54,7 +54,7 @@ def test_validation(): # If you want to test that even init of the VM works, set ``destroy`` to True, it will destroy the previous one as well. # Set to ``False`` by default to bootup time and bandwidth. def test_vagrant(temp_repo_func, destroy=False): - if os.environ.get("TRAVIS", False): + if os.environ.get("CI", False): pytest.skip("Vagrant doesn't work on Travis") backend = VagrantBackend(verbosity=2, use_registry_name="docker.io/mikicz/arca-test", @@ -78,7 +78,7 @@ def test_vagrant(temp_repo_func, destroy=False): # branch branch - return unicode temp_repo_func.repo.create_head("branch") temp_repo_func.repo.branches.branch.checkout() - temp_repo_func.file_path.write_text(SECOND_RETURN_STR_FUNCTION) + temp_repo_func.file_path.write_text(SECOND_RETURN_STR_FUNCTION, encoding="utf-8") temp_repo_func.repo.index.add([str(temp_repo_func.file_path)]) temp_repo_func.repo.index.commit("Test unicode on a separate branch")