Using Debian instead of Alpine, not running tasks as root

Author: mikicz

arca/__init__.py

@@ -7,4 +7,4 @@
 
 __all__ = ["Arca", "BaseBackend", "VenvBackend", "DockerBackend", "Result", "Task", "CurrentEnvironmentBackend",
            "RequirementsStrategy", "VagrantBackend"]
-__version__ = "0.1.1"
+__version__ = "0.2.0a0"

arca/backend/docker.py

@@ -33,7 +33,7 @@ class DockerBackend(BaseBackend):
 
     * **python_version** - set a specific version, current env. python version by default
     * **keep_container_running** - stop the container right away (default) or keep it running
-    * **apk_dependencies** - a list of dependencies to install via alpine
+    * **apt_dependencies** - a list of dependencies to install via ``apt-get``
     * **disable_pull** - build all locally
     * **inherit_image** - instead of using the default base Arca image, use this one
     * **use_registry_name** - use this registry to store images with requirements and dependencies
@@ -42,7 +42,7 @@ class DockerBackend(BaseBackend):
 
     python_version = LazySettingProperty(default=None)
     keep_container_running = LazySettingProperty(default=False)
-    apk_dependencies = LazySettingProperty(default=None)
+    apt_dependencies = LazySettingProperty(default=None)
     disable_pull = LazySettingProperty(default=False)  # so the build can be tested
     inherit_image = LazySettingProperty(default=None)
     use_registry_name = LazySettingProperty(default=None)
@@ -60,8 +60,11 @@ class DockerBackend(BaseBackend):
 
     INSTALL_DEPENDENCIES = """
         FROM {name}:{tag}
-        RUN apk update
-        RUN apk add --no-cache {dependencies}
+        USER root
+        RUN apt-get update && \
+            apt-get install --no-install-recommends -y --autoremove {dependencies} && \
+            apt-get clean
+        USER arca
         CMD bash -i
     """
 
@@ -80,7 +83,7 @@ def validate_settings(self):
 
         * Checks ``inherit_image`` format.
         * Checks ``use_registry_name`` format.
-        * Checks that ``apk_dependencies`` is not set when ``inherit_image`` is set.
+        * Checks that ``apt_dependencies`` is not set when ``inherit_image`` is set.
 
         :raise ArcaMisconfigured: If some of the settings aren't valid.
         """
@@ -118,17 +121,17 @@ def check_docker_access(self):
             raise BuildError("Docker is not running or the current user doesn't have permissions to access docker.")
 
     def get_dependencies(self) -> Optional[List[str]]:
-        """ Returns the ``apk_dependencies`` setting to a standardized format.
+        """ Returns the ``apt_dependencies`` setting to a standardized format.
 
         :raise ArcaMisconfigured: if the dependencies can't be converted into a list of strings
         :return: List of dependencies, ``None`` if there are none.
         """
 
-        if not self.apk_dependencies:
+        if not self.apt_dependencies:
             return None
 
         try:
-            dependencies = list([str(x).strip() for x in self.apk_dependencies])
+            dependencies = list([str(x).strip() for x in self.apt_dependencies])
         except (TypeError, ValueError):
             raise ArcaMisconfigured("Apk dependencies can't be converted into a list of strings")
 
@@ -303,23 +306,34 @@ def get_arca_base(self, pull=True):
 
         pyenv_installer = "https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer"
         dockerfile = f"""
-            FROM alpine:3.5
-            RUN apk add --no-cache curl bash git nano g++ make jpeg-dev zlib-dev ca-certificates openssl-dev \
-                                   readline-dev bzip2-dev sqlite-dev ncurses-dev linux-headers build-base \
-                                   openssh
-
-            RUN curl -L {pyenv_installer} -o /pyenv-installer && \
-                  touch /root/.bashrc && \
-                  /bin/ln -s /root/.bashrc /root/.bash_profile && \
-                  /bin/bash /pyenv-installer && \
-                  rm /pyenv-installer && \
-                  echo 'export PYENV_ROOT="$HOME/.pyenv"' >> ~/.bash_profile && \
-                  echo 'export PATH="$PYENV_ROOT/bin:$PATH"' >> ~/.bash_profile && \
-                  echo 'eval "$(pyenv init -)"' >> ~/.bash_profile
-
-            ENV HOME  /root
-            ENV PYENV_ROOT $HOME/.pyenv
-            ENV PATH $PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH
+            FROM debian:stretch-slim
+            RUN apt-get update && \
+                apt-get install -y make build-essential libssl-dev zlib1g-dev libbz2-dev \
+                                   libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \
+                                   xz-utils tk-dev libffi-dev git && \
+                apt-get clean
+
+            RUN adduser --system --disabled-password --shell /bin/bash arca
+
+            USER arca
+            WORKDIR /home/arca
+            RUN echo "source $HOME/.bash_profile" >> .bashrc
+
+            RUN curl -L {pyenv_installer} -o ~/pyenv-installer && \
+                  /bin/bash ~/pyenv-installer && \
+                  rm ~/pyenv-installer && \
+                  echo 'export PYENV_ROOT="$HOME/.pyenv"' >> /home/arca/.bash_profile && \
+                  echo 'export PATH="$PYENV_ROOT/bin:$PATH"' >> /home/arca/.bash_profile && \
+                  echo 'eval "$(pyenv init -)"' >> /home/arca/.bash_profile
+
+            USER root
+
+            RUN apt-get remove -y --autoremove curl && \
+                apt-get clean -y
+
+            RUN mkdir /srv/scripts
+
+            USER arca
 
             SHELL ["bash", "-lc"]
             CMD bash -i
@@ -341,10 +355,10 @@ def get_dockerfile():
 
             return f"""
                 FROM {base_arca_name}:{base_arca_tag}
-                RUN pyenv update
-                RUN pyenv install {python_version}
+                RUN pyenv update && \
+                    pyenv install {python_version}
                 ENV PYENV_VERSION {python_version}
-                RUN mkdir /srv/scripts
+                ENV PATH "/home/arca/.pyenv/shims:$PATH"
                 CMD bash -i
             """
 
@@ -407,7 +421,7 @@ def get_image_with_installed_dependencies(self, image_name: str,
                                               dependencies: Optional[List[str]]) -> Tuple[str, str]:
         """
         Return name and tag of a image, based on the Arca python image, with installed dependencies defined
-        by ``apk_dependencies``.
+        by ``apt_dependencies``.
 
         :param image_name: Name of the image which will be ultimately used for the image.
         :param dependencies: List of dependencies in the standardized format.
@@ -637,7 +651,11 @@ def start_container(self, image, container_name: str, repo_path: Path):
         :type image: docker.models.images.Image
         :rtype: docker.models.container.Container
         """
-        container = self.client.containers.run(image, command="bash -i", detach=True, tty=True, name=container_name,
+        command = "bash -i"
+        if self.inherit_image:
+            command = "sh -i"
+
+        container = self.client.containers.run(image, command=command, detach=True, tty=True, name=container_name,
                                                working_dir=str((Path("/srv/data") / self.cwd).resolve()),
                                                auto_remove=True)
 

docs/backends.rst

@@ -97,7 +97,8 @@ with ``docker`` (see `documentation <https://docs.docker.com/install/linux/linux
 This backend firstly creates an image with requirements and dependencies installed so the installation only runs one.
 By default the images are based on `custom images <https://hub.docker.com/r/mikicz/arca/tags/>`_, which have Python
 and several build tools pre-installed.
-These images are based on ``alpine`` and use `pyenv <https://github.com/pyenv/pyenv>`_ to install Python.
+These images are based on ``debian`` (slim ``stretch`` version) and use `pyenv <https://github.com/pyenv/pyenv>`_
+to install Python.
 You can specify you want to base your images on a different image with the ``inherit_image`` setting.
 
 Once arca has an image with the requirements installed, it launches a container for each task and
@@ -120,9 +121,9 @@ Settings:
   but only CPython 3.6 has been tested. The default is the Python version of the current environment.
   This setting is ignored if ``inherit_image`` is set.
 * **keep_container_running**: When ``True``, containers aren't killed once the task finishes. Default is ``False``.
-* **apk_dependencies**: For some python libraries, system dependencies are required,
+* **apt_dependencies**: For some python libraries, system dependencies are required,
   for example ``libxml2-dev`` and ``libxslt-dev`` are needed for ``lxml``.
-  With this settings you can specify a list of system dependencies that will be installed via alpine ``apk``.
+  With this settings you can specify a list of system dependencies that will be installed via debian ``apt-get``.
   This setting is ignored if ``inherit_image`` is set since arca can't
   determined how to install requirements on an unknown system.
 * **disable_pull**: Disable pulling prebuilt arca images from Docker Hub and build even the base images locally.

setup.py

@@ -20,7 +20,7 @@ def long_description():
 
 setup(
     name="arca",
-    version="0.1.1",
+    version="0.2.0a0",
     author="Mikuláš Poul",
     author_email="[email protected]",
     description="A library for running Python functions (callables) from git repositories "

tests/common.py

@@ -37,17 +37,17 @@ def return_str_function():
 """
 
 RETURN_PYTHON_VERSION_FUNCTION = """
-import sys
+import platform
 
 def return_python_version():
-    return "{}.{}.{}".format(sys.version_info.major, sys.version_info.minor, sys.version_info.micro)
+    return platform.python_version()
 """
 
-RETURN_FREETYPE_VERSION = """
-import freetype
+RETURN_ALSAAUDIO_INSTALLED = """
+import alsaaudio
 
-def return_freetype_version():
-    return freetype.version()
+def return_alsaaudio_installed():
+    return alsaaudio is not None
 """
 
 RETURN_PLATFORM = """

tests/test_docker.py

@@ -1,9 +1,11 @@
+import platform
+
 import pytest
 
 from arca import Arca, DockerBackend, Task
 from arca.exceptions import ArcaMisconfigured, PushToRegistryError
 from common import (RETURN_COLORAMA_VERSION_FUNCTION, BASE_DIR, RETURN_PLATFORM,
-                    RETURN_PYTHON_VERSION_FUNCTION, RETURN_FREETYPE_VERSION)
+                    RETURN_PYTHON_VERSION_FUNCTION, RETURN_ALSAAUDIO_INSTALLED)
 
 
 def test_keep_container_running(temp_repo_func):
@@ -35,7 +37,7 @@ def test_keep_container_running(temp_repo_func):
     assert count_after_stop == container_count
 
 
-@pytest.mark.parametrize("python_version", ["3.6.0", "3.6.3"])
+@pytest.mark.parametrize("python_version", ["3.6.0", platform.python_version()])
 def test_python_version(temp_repo_func, python_version):
     backend = DockerBackend(verbosity=2, python_version=python_version)
 
@@ -49,25 +51,25 @@ def test_python_version(temp_repo_func, python_version):
     assert arca.run(temp_repo_func.url, temp_repo_func.branch, task).output == python_version
 
 
-def test_apk_dependencies(temp_repo_func):
-    backend = DockerBackend(verbosity=2, apk_dependencies=["freetype-dev"])
+def test_apt_dependencies(temp_repo_func):
+    backend = DockerBackend(verbosity=2, apt_dependencies=["libasound2-dev"])
 
     arca = Arca(backend=backend, base_dir=BASE_DIR)
 
     requirements_path = temp_repo_func.repo_path / "requirements.txt"
-    requirements_path.write_text("freetype-py")
+    requirements_path.write_text("pyalsaaudio==0.8.4")
 
-    temp_repo_func.file_path.write_text(RETURN_FREETYPE_VERSION)
+    temp_repo_func.file_path.write_text(RETURN_ALSAAUDIO_INSTALLED)
     temp_repo_func.repo.index.add([str(temp_repo_func.file_path), str(requirements_path)])
     temp_repo_func.repo.index.commit("Added requirements, changed to lxml")
 
-    # ``import freetype`` raises an error if the library ``freetype-dev`` is not installed
-    task = Task("test_file:return_freetype_version")
+    # pyalsaaudio can't be installed if libasound2-dev is missing
+    task = Task("test_file:return_alsaaudio_installed")
     assert arca.run(temp_repo_func.url, temp_repo_func.branch, task).output
 
 
 def test_inherit_image(temp_repo_func):
-    backend = DockerBackend(verbosity=2, inherit_image="python:3.6")
+    backend = DockerBackend(verbosity=2, inherit_image="python:alpine3.6")
 
     arca = Arca(backend=backend, base_dir=BASE_DIR)
     task = Task("test_file:return_str_function")
@@ -80,8 +82,8 @@ def test_inherit_image(temp_repo_func):
 
     task = Task("test_file:return_platform")
 
-    # alpine is the default, dist() returns ('', '', '') on - so this fails when the default image is used
-    assert arca.run(temp_repo_func.url, temp_repo_func.branch, task).output == "debian"
+    # debian is the default, alpine dist() returns ('', '', '') on - so this fails when the default image is used
+    assert arca.run(temp_repo_func.url, temp_repo_func.branch, task).output != "debian"
 
     requirements_path = temp_repo_func.repo_path / backend.requirements_location
     requirements_path.write_text("colorama==0.3.9")
@@ -161,6 +163,6 @@ def test_push_to_registry_fail(temp_repo_func):
 
 
 def test_inherit_image_with_dependecies():
-    backend = DockerBackend(inherit_image="python:alpine3.6", apk_dependencies=["freetype-dev"])
+    backend = DockerBackend(inherit_image="python:alpine3.6", apt_dependencies=["libasound2-dev"])
     with pytest.raises(ArcaMisconfigured):
         Arca(backend=backend)