Merge pull request #848 from manics/abstractengine

[MRG] Define an interface for Container engines

Author: yuvipanda

repo2docker/__main__.py

@@ -2,8 +2,8 @@
 import sys
 import os
 import logging
-import docker
 from .app import Repo2Docker
+from .engine import BuildError, ImageLoadError
 from . import __version__
 from .utils import validate_and_generate_port_mapping, is_valid_docker_image_name
 
@@ -217,6 +217,8 @@ def get_argparser():
         "--cache-from", action="append", default=[], help=Repo2Docker.cache_from.help
     )
 
+    argparser.add_argument("--engine", help="Name of the container engine")
+
     return argparser
 
 
@@ -351,6 +353,9 @@ def make_r2d(argv=None):
     if args.cache_from:
         r2d.cache_from = args.cache_from
 
+    if args.engine:
+        r2d.engine = args.engine
+
     r2d.environment = args.environment
 
     # if the source exists locally we don't want to delete it at the end
@@ -371,12 +376,12 @@ def main():
     r2d.initialize()
     try:
         r2d.start()
-    except docker.errors.BuildError as e:
+    except BuildError as e:
         # This is only raised by us
         if r2d.log_level == logging.DEBUG:
             r2d.log.exception(e)
         sys.exit(1)
-    except docker.errors.ImageLoadError as e:
+    except ImageLoadError as e:
         # This is only raised by us
         if r2d.log_level == logging.DEBUG:
             r2d.log.exception(e)

repo2docker/app.py

@@ -11,15 +11,14 @@
 import sys
 import logging
 import os
+import entrypoints
 import getpass
 import shutil
 import tempfile
 import time
 
-import docker
+from .engine import BuildError, ContainerEngineException, ImageLoadError
 from urllib.parse import urlparse
-from docker.utils import kwargs_from_env
-from docker.errors import DockerException
 import escapism
 from pythonjsonlogger import jsonlogger
 
@@ -382,6 +381,33 @@ def _user_name_default(self):
         config=True,
     )
 
+    engine = Unicode(
+        "docker",
+        config=True,
+        help="""
+        Name of the container engine.
+
+        Defaults to 'docker'.
+        """,
+    )
+
+    def get_engine(self):
+        """Return an instance of the container engine.
+
+        Currently no arguments are passed to the engine constructor.
+        """
+        engines = entrypoints.get_group_named("repo2docker.engines")
+        try:
+            entry = engines[self.engine]
+        except KeyError:
+            raise ContainerEngineException(
+                "Container engine '{}' not found. Available engines: {}".format(
+                    self.engine, ",".join(engines.keys())
+                )
+            )
+        engine_class = entry.load()
+        return engine_class(parent=self)
+
     def fetch(self, url, ref, checkout_path):
         """Fetch the contents of `url` and place it in `checkout_path`.
 
@@ -474,13 +500,18 @@ def initialize(self):
 
     def push_image(self):
         """Push docker image to registry"""
-        client = docker.APIClient(version="auto", **kwargs_from_env())
+        client = self.get_engine()
         # Build a progress setup for each layer, and only emit per-layer
         # info every 1.5s
         progress_layers = {}
         layers = {}
         last_emit_time = time.time()
-        for chunk in client.push(self.output_image_spec, stream=True):
+        for chunk in client.push(self.output_image_spec):
+            if client.string_output:
+                self.log.info(chunk, extra=dict(phase="pushing"))
+                continue
+            # else this is Docker output
+
             # each chunk can be one or more lines of json events
             # split lines here in case multiple are delivered at once
             for line in chunk.splitlines():
@@ -492,7 +523,7 @@ def push_image(self):
                     continue
                 if "error" in progress:
                     self.log.error(progress["error"], extra=dict(phase="failed"))
-                    raise docker.errors.ImageLoadError(progress["error"])
+                    raise ImageLoadError(progress["error"])
                 if "id" not in progress:
                     continue
                 # deprecated truncated-progress data
@@ -528,7 +559,7 @@ def start_container(self):
 
         Returns running container
         """
-        client = docker.from_env(version="auto")
+        client = self.get_engine()
 
         docker_host = os.environ.get("DOCKER_HOST")
         if docker_host:
@@ -565,11 +596,8 @@ def start_container(self):
 
         container_volumes = {}
         if self.volumes:
-            api_client = docker.APIClient(
-                version="auto", **docker.utils.kwargs_from_env()
-            )
-            image = api_client.inspect_image(self.output_image_spec)
-            image_workdir = image["ContainerConfig"]["WorkingDir"]
+            image = client.inspect_image(self.output_image_spec)
+            image_workdir = image.config["WorkingDir"]
 
             for k, v in self.volumes.items():
                 container_volumes[os.path.abspath(k)] = {
@@ -580,15 +608,14 @@ def start_container(self):
         run_kwargs = dict(
             publish_all_ports=self.all_ports,
             ports=ports,
-            detach=True,
             command=run_cmd,
             volumes=container_volumes,
             environment=self.environment,
         )
 
         run_kwargs.update(self.extra_run_kwargs)
 
-        container = client.containers.run(self.output_image_spec, **run_kwargs)
+        container = client.run(self.output_image_spec, **run_kwargs)
 
         while container.status == "created":
             time.sleep(0.5)
@@ -611,7 +638,7 @@ def wait_for_container(self, container):
             if container.status == "running":
                 self.log.info("Stopping container...\n", extra=dict(phase="running"))
                 container.kill()
-            exit_code = container.attrs["State"]["ExitCode"]
+            exit_code = container.exitcode
 
             container.wait()
 
@@ -645,12 +672,11 @@ def find_image(self):
         if self.dry_run:
             return False
         # check if we already have an image for this content
-        client = docker.APIClient(version="auto", **kwargs_from_env())
+        client = self.get_engine()
         for image in client.images():
-            if image["RepoTags"] is not None:
-                for tag in image["RepoTags"]:
-                    if tag == self.output_image_spec + ":latest":
-                        return True
+            for tag in image.tags:
+                if tag == self.output_image_spec + ":latest":
+                    return True
         return False
 
     def build(self):
@@ -660,12 +686,9 @@ def build(self):
         # Check if r2d can connect to docker daemon
         if not self.dry_run:
             try:
-                docker_client = docker.APIClient(version="auto", **kwargs_from_env())
-            except DockerException as e:
-                self.log.error(
-                    "\nDocker client initialization error: %s.\nCheck if docker is running on the host.\n",
-                    e,
-                )
+                docker_client = self.get_engine()
+            except ContainerEngineException as e:
+                self.log.error("\nContainer engine initialization error: %s\n", e)
                 self.exit(1)
 
         # If the source to be executed is a directory, continue using the
@@ -751,11 +774,14 @@ def build(self):
                         self.cache_from,
                         self.extra_build_kwargs,
                     ):
-                        if "stream" in l:
+                        if docker_client.string_output:
+                            self.log.info(l, extra=dict(phase="building"))
+                        # else this is Docker output
+                        elif "stream" in l:
                             self.log.info(l["stream"], extra=dict(phase="building"))
                         elif "error" in l:
                             self.log.info(l["error"], extra=dict(phase="failure"))
-                            raise docker.errors.BuildError(l["error"], build_log="")
+                            raise BuildError(l["error"])
                         elif "status" in l:
                             self.log.info(
                                 "Fetching base image...\r", extra=dict(phase="building")

repo2docker/buildpacks/base.py

@@ -607,9 +607,6 @@ def _filter_tar(tar):
             tag=image_spec,
             custom_context=True,
             buildargs=build_args,
-            decode=True,
-            forcerm=True,
-            rm=True,
             container_limits=limits,
             cache_from=cache_from,
         )

repo2docker/buildpacks/docker.py

@@ -50,9 +50,6 @@ def build(
             dockerfile=self.binder_path(self.dockerfile),
             tag=image_spec,
             buildargs=build_args,
-            decode=True,
-            forcerm=True,
-            rm=True,
             container_limits=limits,
             cache_from=cache_from,
         )

repo2docker/docker.py

@@ -0,0 +1,119 @@
+"""
+Docker container engine for repo2docker
+"""
+
+import docker
+from .engine import Container, ContainerEngine, ContainerEngineException, Image
+
+
+class DockerContainer(Container):
+    def __init__(self, container):
+        self._c = container
+
+    def reload(self):
+        return self._c.reload()
+
+    def logs(self, *, stream=False):
+        return self._c.logs(stream=stream)
+
+    def kill(self, *, signal="KILL"):
+        return self._c.kill(signal=signal)
+
+    def remove(self):
+        return self._c.remove()
+
+    def stop(self, *, timeout=10):
+        return self._c.stop(timeout=timeout)
+
+    def wait(self):
+        return self._c.wait()
+
+    @property
+    def exitcode(self):
+        return self._c.attrs["State"]["ExitCode"]
+
+    @property
+    def status(self):
+        return self._c.status
+
+
+class DockerEngine(ContainerEngine):
+    """
+    https://docker-py.readthedocs.io/en/4.2.0/api.html#module-docker.api.build
+    """
+
+    string_output = False
+
+    def __init__(self, *, parent):
+        super().__init__(parent=parent)
+        try:
+            self._apiclient = docker.APIClient(
+                version="auto", **docker.utils.kwargs_from_env()
+            )
+        except docker.errors.DockerException as e:
+            raise ContainerEngineException("Check if docker is running on the host.", e)
+
+    def build(
+        self,
+        *,
+        buildargs=None,
+        cache_from=None,
+        container_limits=None,
+        tag="",
+        custom_context=False,
+        dockerfile="",
+        fileobj=None,
+        path="",
+        **kwargs,
+    ):
+        return self._apiclient.build(
+            buildargs=buildargs,
+            cache_from=cache_from,
+            container_limits=container_limits,
+            forcerm=True,
+            rm=True,
+            tag=tag,
+            custom_context=custom_context,
+            decode=True,
+            dockerfile=dockerfile,
+            fileobj=fileobj,
+            path=path,
+            **kwargs,
+        )
+
+    def images(self):
+        images = self._apiclient.images()
+        return [Image(tags=image["RepoTags"]) for image in images]
+
+    def inspect_image(self, image):
+        image = self._apiclient.inspect_image(image)
+        return Image(tags=image["RepoTags"], config=image["ContainerConfig"])
+
+    def push(self, image_spec):
+        return self._apiclient.push(image_spec, stream=True)
+
+    def run(
+        self,
+        image_spec,
+        *,
+        command=None,
+        environment=None,
+        ports=None,
+        publish_all_ports=False,
+        remove=False,
+        volumes=None,
+        **kwargs,
+    ):
+        client = docker.from_env(version="auto")
+        container = client.containers.run(
+            image_spec,
+            command=command,
+            environment=(environment or []),
+            detach=True,
+            ports=(ports or {}),
+            publish_all_ports=publish_all_ports,
+            remove=remove,
+            volumes=(volumes or {}),
+            **kwargs,
+        )
+        return DockerContainer(container)