Using extras_require for requirements for Docker and Vagrant backends

Author: mikicz

.travis.yml

@@ -29,7 +29,8 @@ after_success:
 - codecov
 branches:
   only:
-    master
+  - master
+  - /release-v[0-9]+/
 deploy:
   provider: script
   skip_cleanup: true

README.rst

@@ -52,11 +52,25 @@ To install the last stable version:
 
   python -m pip install arca
 
+If you want to use the Docker backend:
+
+.. code-block:: bash
+
+  python -m  pip install arca[docker]
+
+Or if you want to use the Vagrant backend:
+
+.. code-block:: bash
+
+  python -m pip install arca[vagrant]
+
 Or if you wish to install the upstream version:
 
 .. code-block:: bash
 
   python -m pip install git+https://github.com/mikicz/arca.git#egg=arca
+  python -m pip install git+https://github.com/mikicz/arca.git#egg=arca[docker]
+  python -m pip install git+https://github.com/mikicz/arca.git#egg=arca[vagrant]
 
 Example
 +++++++

arca/backend/docker.py

@@ -9,10 +9,12 @@
 from pathlib import Path
 from typing import Optional, List, Tuple, Union, Callable
 
-import docker
-import docker.errors
-from docker.models.containers import Container, ExecResult
-from docker.models.images import Image
+try:
+    import docker
+    import docker.errors
+except ImportError:
+    docker = None
+
 from git import Repo
 from requests.exceptions import ConnectionError
 
@@ -66,6 +68,9 @@ class DockerBackend(BaseBackend):
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
+        if docker is None:
+            raise ArcaMisconfigured(ArcaMisconfigured.PACKAGE_MISSING.format("docker"))
+
         self._containers = set()
         self.client = None
 
@@ -367,12 +372,14 @@ def get_inherit_image(self) -> Tuple[str, str]:
 
     def build_image_from_inherited_image(self, image_name: str, image_tag: str,
                                          build_context: Path,
-                                         requirements_file: Optional[Path]) -> Image:
+                                         requirements_file: Optional[Path]):
         """
         Builds a image with installed requirements from the inherited image. (Or just tags the image
         if there are no requirements.)
 
         See :meth:`build_image` for parameters descriptions.
+
+        :rtype: docker.models.images.Image
         """
 
         base_name, base_tag = self.get_inherit_image()
@@ -429,7 +436,7 @@ def install_dependencies_dockerfile():
     def build_image(self, image_name: str, image_tag: str,
                     build_context: Path,
                     requirements_file: Optional[Path],
-                    dependencies: Optional[List[str]]) -> Image:
+                    dependencies: Optional[List[str]]):
         """ Builds an image for specific requirements and dependencies, based on the settings.
 
         :param image_name: How the image should be named
@@ -438,6 +445,7 @@ def build_image(self, image_name: str, image_tag: str,
         :param requirements_file: Path to the requirements file in the repository (or ``None`` if it doesn't exist)
         :param dependencies: List of dependencies (in the formalized format)
         :return: The Image instance.
+        :rtype: docker.models.images.Image
         """
         if self.inherit_image is not None:
             return self.build_image_from_inherited_image(image_name, image_tag, build_context, requirements_file)
@@ -491,9 +499,10 @@ def install_requirements_dockerfile():
 
             return self.get_image(image_name, image_tag)
 
-    def push_to_registry(self, image: Image, image_tag: str):
+    def push_to_registry(self, image, image_tag: str):
         """ Pushes a local image to a registry based on the ``use_registry_name`` setting.
 
+        :type image: docker.models.images.Image
         :raise PushToRegistryError: If the push fails.
         """
         # already tagged, so it's already pushed
@@ -530,21 +539,24 @@ def image_exists(self, image_name, image_tag):
         except docker.errors.ImageNotFound:
             return False
 
-    def get_image(self, image_name, image_tag) -> Image:
+    def get_image(self, image_name, image_tag):
         """ Returns a :class:`Image <docker.models.images.Image>` instance for the provided name and tag.
+
+        :rtype: docker.models.images.Image
         """
         return self.client.images.get(f"{image_name}:{image_tag}")
 
-    def try_pull_image_from_registry(self, image_name, image_tag) -> Optional[Image]:
+    def try_pull_image_from_registry(self, image_name, image_tag):
         """
         Tries to pull a image with the tag ``image_tag`` from registry set by ``use_registry_name``.
         After the image is pulled, it's tagged with ``image_name``:``image_tag`` so lookup can
         be made locally next time.
 
         :return: A :class:`Image <docker.models.images.Image>` instance if the image exists, ``None`` otherwise.
+        :rtype: Optional[docker.models.images.Image]
         """
         try:
-            image: Image = self.client.images.pull(self.use_registry_name, image_tag)
+            image = self.client.images.pull(self.use_registry_name, image_tag)
         except (docker.errors.ImageNotFound, docker.errors.NotFound):  # the image doesn't exist
             logger.info("Tried to pull %s:%s from a registry, not found", self.use_registry_name, image_tag)
             return None
@@ -557,11 +569,12 @@ def try_pull_image_from_registry(self, image_name, image_tag) -> Optional[Image]
 
         return image
 
-    def container_running(self, container_name) -> Optional[Container]:
+    def container_running(self, container_name):
         """
         Finds out if a container with name ``container_name`` is running.
 
         :return: :class:`Container <docker.models.containers.Container>` if it's running, ``None`` otherwise.
+        :rtype: Optional[docker.models.container.Container]
         """
         filters = {
             "name": container_name,
@@ -618,8 +631,11 @@ def tar_task_definition(self, name: str, contents: str) -> bytes:
 
         return tarstream.getvalue()
 
-    def start_container(self, image: Image, container_name: str, repo_path: Path) -> Container:
+    def start_container(self, image, container_name: str, repo_path: Path):
         """ Starts a container with the image and name ``container_name`` and copies the repository into the container.
+
+        :type image: docker.models.images.Image
+        :rtype: docker.models.container.Container
         """
         container = self.client.containers.run(image, command="bash -i", detach=True, tty=True, name=container_name,
                                                working_dir=str((Path("/srv/data") / self.cwd).resolve()),
@@ -631,7 +647,7 @@ def start_container(self, image: Image, container_name: str, repo_path: Path) ->
 
         return container
 
-    def get_image_for_repo(self, repo: str, branch: str, git_repo: Repo, repo_path: Path) -> Image:
+    def get_image_for_repo(self, repo: str, branch: str, git_repo: Repo, repo_path: Path):
         """
         Returns an image for the specific repo (based on settings and requirements).
 
@@ -641,6 +657,8 @@ def get_image_for_repo(self, repo: str, branch: str, git_repo: Repo, repo_path:
         4. Pushes the image to registry so the image is available next time (if ``registry_pull_only`` is not set)
 
         See :meth:`run` for parameters descriptions.
+
+        :rtype: docker.models.images.Image
         """
         requirements_file = self.get_requirements_file(repo_path)
         dependencies = self.get_dependencies()
@@ -703,7 +721,7 @@ def run(self, repo: str, branch: str, task: Task, git_repo: Repo, repo_path: Pat
 
         container.put_archive("/srv/scripts", self.tar_task_definition(task_filename, task_json))
 
-        res: Optional[ExecResult] = None
+        res = None
 
         try:
             res = container.exec_run(["python",

arca/backend/vagrant.py

@@ -5,7 +5,6 @@
 from textwrap import dedent
 from uuid import uuid4
 
-from fabric import api
 from git import Repo
 
 from arca.exceptions import ArcaMisconfigured, BuildError
@@ -15,17 +14,6 @@
 from .docker import DockerBackend
 
 
-@api.task
-def run_script(container_name, definition_filename):
-    """ Sequence to run inside the VM, copies data and script to the container and runs the script.
-    """
-    api.run(f"docker exec {container_name} mkdir -p /srv/scripts")
-    api.run(f"docker cp /srv/data {container_name}:/srv")
-    api.run(f"docker cp /vagrant/runner.py {container_name}:/srv/scripts/")
-    api.run(f"docker cp /vagrant/{definition_filename} {container_name}:/srv/scripts/")
-    return api.run(f"docker exec -t {container_name} python /srv/scripts/runner.py /srv/scripts/{definition_filename}")
-
-
 class VagrantBackend(DockerBackend):
     """ Uses Docker in Vagrant.
 
@@ -53,6 +41,27 @@ class VagrantBackend(DockerBackend):
     quiet = LazySettingProperty(key="quiet", default=True, convert=bool)
     destroy = LazySettingProperty(key="destroy", default=True, convert=bool)
 
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        try:
+            import docker  # noqa: F401
+        except ImportError:
+            raise ArcaMisconfigured(ArcaMisconfigured.PACKAGE_MISSING.format("docker"))
+
+        try:
+            import vagrant
+        except ImportError:
+            raise ArcaMisconfigured(ArcaMisconfigured.PACKAGE_MISSING.format("python-vagrant"))
+
+        try:
+            import fabric  # noqa: F401
+        except ImportError:
+            raise ArcaMisconfigured(ArcaMisconfigured.PACKAGE_MISSING.format("fabric3"))
+
+        if vagrant.get_vagrant_executable() is None:
+            raise ArcaMisconfigured("Vagrant executable is not accessible!")
+
     def validate_settings(self):
         """ Runs :meth:`arca.DockerBackend.validate_settings` and checks extra:
 
@@ -74,15 +83,6 @@ def validate_settings(self):
         if self.registry_pull_only:
             raise ArcaMisconfigured("Push must be enabled for VagrantBackend")
 
-    def check_vagrant_access(self):
-        """
-        :raise BuildError: If Vagrant is not installed.
-        """
-        from vagrant import get_vagrant_executable
-
-        if get_vagrant_executable() is None:
-            raise BuildError("Vagrant executable is not accessible!")
-
     def get_vagrant_file_location(self, repo: str, branch: str, git_repo: Repo, repo_path: Path) -> Path:
         """ Returns a directory where Vagrantfile should be. Based on repo, branch and tag of the used docker image.
         """
@@ -142,13 +142,33 @@ def create_vagrant_file(self, repo: str, branch: str, git_repo: Repo, repo_path:
 
         (vagrant_file.parent / "runner.py").write_text(self._arca.RUNNER.read_text())
 
+    @property
+    def fabric_task(self):
+        """ Returns a fabric task which executes the script in the Vagrant VM
+        """
+        from fabric import api
+
+        @api.task
+        def run_script(container_name, definition_filename):
+            """ Sequence to run inside the VM, copies data and script to the container and runs the script.
+            """
+            api.run(f"docker exec {container_name} mkdir -p /srv/scripts")
+            api.run(f"docker cp /srv/data {container_name}:/srv")
+            api.run(f"docker cp /vagrant/runner.py {container_name}:/srv/scripts/")
+            api.run(f"docker cp /vagrant/{definition_filename} {container_name}:/srv/scripts/")
+            return api.run(" ".join([
+                "docker", "exec", "-t", container_name,
+                "python", "/srv/scripts/runner.py", f"/srv/scripts/{definition_filename}",
+            ]))
+
+        return run_script
+
     def run(self, repo: str, branch: str, task: Task, git_repo: Repo, repo_path: Path):
         """ Gets or creates Vagrantfile, starts up a VM with it, executes Fabric script over SSH, returns result.
         """
         # importing here, prints out warning when vagrant is missing even when the backend is not used otherwise
         from vagrant import Vagrant, make_file_cm
-
-        self.check_vagrant_access()
+        from fabric import api
 
         vagrant_file = self.get_vagrant_file_location(repo, branch, git_repo, repo_path)
 
@@ -191,7 +211,7 @@ def run(self, repo: str, branch: str, task: Task, git_repo: Repo, repo_path: Pat
             api.output.everything = True
 
         try:
-            res = api.execute(run_script, container_name=container_name, definition_filename=task_filename)
+            res = api.execute(self.fabric_task, container_name=container_name, definition_filename=task_filename)
 
             return Result(json.loads(res[vagrant.user_hostname_port()].stdout))
         except BuildError:  # can be raised by  :meth:`Result.__init__`

arca/exceptions.py

@@ -9,6 +9,9 @@ class ArcaMisconfigured(ValueError, ArcaException):
     """ An exception for all cases of misconfiguration.
     """
 
+    PACKAGE_MISSING = "Couldn't import package '{}' that is required for this backend. " \
+                      "Did you install the extra requirements for this backend?"
+
 
 class TaskMisconfigured(ValueError, ArcaException):
     """ Raised if Task is incorrectly defined.

docs/backends.rst

@@ -128,7 +128,7 @@ Settings:
 * **disable_pull**: Disable pulling prebuilt arca images from Docker Hub and build even the base images locally.
 * **inherit_image**: If you don't wish to use the arca images you can specify what image should be used instead.
 * **use_registry_name**: Uses this registry to store images with installed requirements and dependencies to,
-tries to pull image from the registry before building it locally to save time.
+  tries to pull image from the registry before building it locally to save time.
 * **registry_pull_only**: Disables pushing to registry.
 
 (possible settings prefixes: ``ARCA_DOCKER_BACKEND_`` and ``ARCA_BACKEND_``)

docs/install.rst

@@ -24,8 +24,22 @@ To install the last stable version:
 
   python -m pip install arca
 
+If you want to use the Docker backend:
+
+.. code-block:: bash
+
+  python -m  pip install arca[docker]
+
+Or if you want to use the Vagrant backend:
+
+.. code-block:: bash
+
+  python -m pip install arca[vagrant]
+
 Or if you wish to install the upstream version:
 
 .. code-block:: bash
 
   python -m pip install git+https://github.com/mikicz/arca.git#egg=arca
+  python -m pip install git+https://github.com/mikicz/arca.git#egg=arca[docker]
+  python -m pip install git+https://github.com/mikicz/arca.git#egg=arca[vagrant]

setup.py

@@ -34,12 +34,19 @@ def long_description():
         "gitpython==2.1.9",
         "dogpile.cache==0.6.4",
         "requests",
-        "docker~=3.1.0",
-        "python-vagrant",
-        "fabric3",
         "entrypoints>=0.2.3",
         "cached-property",
     ],
+    extras_require={
+        "docker": [
+            "docker~=3.1.0",
+        ],
+        "vagrant": [
+            "docker~=3.1.0",
+            "python-vagrant",
+            "fabric3",
+        ],
+    },
     classifiers=[
         "Development Status :: 3 - Alpha",
         "Intended Audience :: Developers",

tests/test_vagrant.py

@@ -8,8 +8,9 @@
 
 
 def test_validation():
-    """ These tests work on Travis
-    """
+    if os.environ.get("TRAVIS", False):
+        pytest.skip("Vagrant doesn't work on Travis")
+
     backend = VagrantBackend()
 
     # VagrantBackend requires `push_to_registry`