Timeout for installing requirements

Author: mikicz

arca/backend/base.py

@@ -21,12 +21,16 @@ class BaseBackend:
 
     * **requirements_location**: Relative path to the requirements file in the target repositories.
       (default is ``requirements.txt``)
-    * **cwd**: Relative path to the required working directory. (default is ``""``, the root of the repo)
+    * **requirements_timeout**: The maximum time in seconds allowed for installing requirements.
+      (default is 5 minutes, 300 seconds)
+    * **cwd**: Relative path to the required working directory.
+      (default is ``""``, the root of the repo)
     """
 
     RUNNER = Path(__file__).parent.parent.resolve() / "_runner.py"
 
     requirements_location: str = LazySettingProperty(default="requirements.txt")
+    requirements_timeout: int = LazySettingProperty(default=300, convert=int)
     cwd: str = LazySettingProperty(default="")
 
     def __init__(self, **settings):

arca/backend/current_environment.py

@@ -6,7 +6,7 @@
 
 from git import Repo
 
-from arca.exceptions import ArcaMisconfigured, RequirementsMismatch, BuildError
+from arca.exceptions import ArcaMisconfigured, RequirementsMismatch, BuildError, BuildTimeoutError
 from arca.utils import LazySettingProperty, logger
 from .base import BaseRunInSubprocessBackend
 
@@ -76,7 +76,11 @@ def install_requirements(self, *, path: Optional[Path] = None, requirements: Opt
 
         process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 
-        [out_stream, err_stream] = process.communicate()
+        try:
+            out_stream, err_stream = process.communicate(timeout=self.requirements_timeout)
+        except subprocess.TimeoutExpired:
+            raise BuildTimeoutError(f"Installing of requirements timeouted after {self.requirements_timeout} seconds.")
+
         out_stream = out_stream.decode("utf-8")
         err_stream = err_stream.decode("utf-8")
 

arca/backend/docker.py

@@ -54,7 +54,11 @@ class DockerBackend(BaseBackend):
     INSTALL_REQUIREMENTS = """
         FROM {name}:{tag}
         ADD {requirements} /srv/requirements.txt
-        RUN pip install -r /srv/requirements.txt
+        RUN if grep -q Alpine /etc/issue; then \
+                timeout -t {timeout} pip install --no-cache-dir -r /srv/requirements.txt; \
+            else \
+                timeout {timeout} pip install --no-cache-dir -r /srv/requirements.txt; \
+            fi
         CMD bash -i
     """
 
@@ -296,6 +300,11 @@ def get_or_build_image(self, name: str, tag: str, dockerfile: Union[str, Callabl
 
                 dockerfile_file.unlink()
         except docker.errors.BuildError as e:
+            for line in e.build_log:
+                if isinstance(line, dict) and line.get("errorDetail") and line["errorDetail"].get("code") in {124, 143}:
+                    raise BuildTimeoutError(f"Installing of requirements timeouted after "
+                                            f"{self.requirements_timeout} seconds.")
+
             logger.exception(e)
             raise
 
@@ -412,7 +421,8 @@ def build_image_from_inherited_image(self, image_name: str, image_tag: str,
         install_requirements_dockerfile = self.INSTALL_REQUIREMENTS.format(
             name=base_name,
             tag=base_tag,
-            requirements=relative_requirements
+            requirements=relative_requirements,
+            timeout=self.requirements_timeout
         )
 
         self.get_or_build_image(image_name, image_tag, install_requirements_dockerfile,
@@ -508,7 +518,8 @@ def install_requirements_dockerfile():
                 return self.INSTALL_REQUIREMENTS.format(
                     name=dependencies_name,
                     tag=dependencies_tag,
-                    requirements=relative_requirements
+                    requirements=relative_requirements,
+                    timeout=self.requirements_timeout
                 )
 
             self.get_or_build_image(image_name, image_tag, install_requirements_dockerfile, build_context=build_context,

arca/backend/venv.py

@@ -1,10 +1,11 @@
+import shutil
 import subprocess
 from pathlib import Path
 from venv import EnvBuilder
 
 from git import Repo
 
-from arca.exceptions import BuildError
+from arca.exceptions import BuildError, BuildTimeoutError
 from arca.utils import logger
 from .base import BaseRunInSubprocessBackend
 
@@ -65,7 +66,14 @@ def get_or_create_venv(self, path: Path) -> Path:
                                             str(requirements_file)],
                                            stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 
-                [out_stream, err_stream] = process.communicate()
+                try:
+                    out_stream, err_stream = process.communicate(timeout=self.requirements_timeout)
+                except subprocess.TimeoutExpired:
+                    shutil.rmtree(venv_path, ignore_errors=True)
+
+                    raise BuildTimeoutError(f"Installing of requirements timeouted after "
+                                            f"{self.requirements_timeout} seconds.")
+
                 out_stream = out_stream.decode("utf-8")
                 err_stream = err_stream.decode("utf-8")
 

docs/backends.rst

@@ -39,6 +39,7 @@ or you can use settings (described in more details in :ref:`configuring`). For e
 As mentioned in :ref:`options`, there are two options common for all backends. (See that section for more details.)
 
 * **requirements_location**
+* **requirements_timeout**
 * **cwd**
 
 .. _backends_cur:

docs/changes.rst

@@ -17,7 +17,8 @@ Changes:
     * **apk_dependencies** changed to **apt_dependencies**, now installing using `apt-get`
 
   * Vagrant backend only creates one VM, instead of multiple -- see its documentation
-  * Added timeout to tasks -- 5 being the default
+  * Added timeout to tasks, 5 seconds by default. Can be set using the argument **timeout** for ``Task``.
+  * Added timeout to installing requirements, 300 seconds by default. Can be set using the **requirements_timeout** configuration option for backends.
 
 0.1.1 (2018-04-23)
 ******************

docs/settings.rst

@@ -71,6 +71,12 @@ This section describes settings that are common for all the backends.
 Tells backends where to look for a requirements file in the repositories, so it must be a relative path. You can set it
 to ``None`` to indicate there are no requirements. The default is ``requirements.txt``.
 
+**requirements_timeout** (`ARCA_BACKEND_REQUIREMENTS_TIMEOUT`)
+
+Tells backends how long the installing of requirements can take, in seconds.
+The default is 120 seconds.
+If the limit is exceeded :class:`BuildTimeoutError <arca.exceptions.BuildTimeoutError>` is raised.
+
 **cwd** (`ARCA_BACKEND_CWD`)
 
 Tells Arca in what working directory the tasks should be launched, so again a relative path.

tests/test_backends.py

@@ -4,9 +4,9 @@
 import pytest
 
 from arca import Arca, VenvBackend, DockerBackend, Task, CurrentEnvironmentBackend
-from common import BASE_DIR, RETURN_COLORAMA_VERSION_FUNCTION, SECOND_RETURN_STR_FUNCTION, \
-    TEST_UNICODE, ARG_STR_FUNCTION, KWARG_STR_FUNCTION, WAITING_FUNCTION
 from arca.exceptions import BuildTimeoutError
+from common import BASE_DIR, RETURN_COLORAMA_VERSION_FUNCTION, SECOND_RETURN_STR_FUNCTION, \
+    TEST_UNICODE, ARG_STR_FUNCTION, KWARG_STR_FUNCTION, WAITING_FUNCTION, RETURN_STR_FUNCTION
 
 
 @pytest.mark.parametrize(
@@ -17,6 +17,9 @@
     ))
 )
 def test_backends(temp_repo_func, backend, requirements_location, file_location):
+    """ Tests the basic stuff around backends, if it can install requirements from more locations,
+        launch stuff with correct cwd, works well with multiple branches, etc
+    """
     if os.environ.get("TRAVIS", False) and backend == VenvBackend:
         pytest.skip("Venv Backend doesn't work on Travis")
 
@@ -86,6 +89,41 @@ def test_backends(temp_repo_func, backend, requirements_location, file_location)
 
     assert arca.run(temp_repo_func.url, temp_repo_func.branch, task).output == "0.3.8"
 
+    # cleanup
+
+    if isinstance(backend, CurrentEnvironmentBackend):
+        backend._uninstall("colorama")
+
+    with pytest.raises(ModuleNotFoundError):
+        import colorama  # noqa
+
+
+@pytest.mark.parametrize(
+    "backend",
+    [CurrentEnvironmentBackend, VenvBackend, DockerBackend]
+)
+def test_advanced_backends(temp_repo_func, backend):
+    """ Tests the more time-intensive stuff, like timeouts or arguments,
+        things multiple for runs with different arguments are not neccessary
+    """
+    if os.environ.get("TRAVIS", False) and backend == VenvBackend:
+        pytest.skip("Venv Backend doesn't work on Travis")
+
+    kwargs = {}
+
+    if backend == DockerBackend:
+        kwargs["disable_pull"] = True
+    if backend == CurrentEnvironmentBackend:
+        kwargs["current_environment_requirements"] = None
+        kwargs["requirements_strategy"] = "install_extra"
+
+    backend = backend(verbosity=2, **kwargs)
+
+    arca = Arca(backend=backend, base_dir=BASE_DIR)
+
+    filepath = temp_repo_func.file_path
+    requirements_path = temp_repo_func.repo_path / backend.requirements_location
+
     filepath.write_text(ARG_STR_FUNCTION)
     temp_repo_func.repo.index.add([str(filepath)])
     temp_repo_func.repo.index.commit("Argument function")
@@ -104,6 +142,7 @@ def test_backends(temp_repo_func, backend, requirements_location, file_location)
         kwargs={"kwarg": TEST_UNICODE}
     )).output == TEST_UNICODE[::-1]
 
+    # test task timeout
     filepath.write_text(WAITING_FUNCTION)
     temp_repo_func.repo.index.add([str(filepath)])
     temp_repo_func.repo.index.commit("Waiting function")
@@ -112,12 +151,19 @@ def test_backends(temp_repo_func, backend, requirements_location, file_location)
     task_3_seconds = Task("test_file:return_str_function", timeout=3)
 
     with pytest.raises(BuildTimeoutError):
-        assert arca.run(temp_repo_func.url, temp_repo_func.branch, task_1_second).output == "Some string"
+        arca.run(temp_repo_func.url, temp_repo_func.branch, task_1_second)
 
     assert arca.run(temp_repo_func.url, temp_repo_func.branch, task_3_seconds).output == "Some string"
 
-    if isinstance(backend, CurrentEnvironmentBackend):
-        backend._uninstall("colorama")
+    # test requirements timeout
+    requirements_path.write_text("scipy")
 
-    with pytest.raises(ModuleNotFoundError):
-        import colorama  # noqa
+    filepath.write_text(RETURN_STR_FUNCTION)
+
+    temp_repo_func.repo.index.add([str(filepath), str(requirements_path)])
+    temp_repo_func.repo.index.commit("Updated requirements to something that takes > 1 second to install")
+
+    arca.backend.requirements_timeout = 1
+
+    with pytest.raises(BuildTimeoutError):
+        arca.run(temp_repo_func.url, temp_repo_func.branch, Task("test_file:return_str_function"))