check for local pip requirements in conda environments

Author: minrk

repo2docker/buildpacks/conda/__init__.py

@@ -6,6 +6,7 @@
 from ruamel.yaml import YAML
 
 from ..base import BaseImage
+from ...utils import is_local_pip_requirement
 
 # pattern for parsing conda dependency line
 PYTHON_REGEX = re.compile(r"python\s*=+\s*([\d\.]*)")
@@ -127,6 +128,50 @@ def get_build_script_files(self):
         files.update(super().get_build_script_files())
         return files
 
+    _environment_yaml = None
+
+    @property
+    def environment_yaml(self):
+        if self._environment_yaml is not None:
+            return self._environment_yaml
+
+        environment_yml = self.binder_path("environment.yml")
+        if not os.path.exists(environment_yml):
+            self._environment_yaml = {}
+            return self._environment_yaml
+
+        with open(environment_yml) as f:
+            env = YAML().load(f)
+            # check if the env file is empty, if so instantiate an empty dictionary.
+            if env is None:
+                env = {}
+            # check if the env file provided a dick-like thing not a list or other data structure.
+            if not isinstance(env, Mapping):
+                raise TypeError(
+                    "environment.yml should contain a dictionary. Got %r" % type(env)
+                )
+            self._environment_yaml = env
+
+        return self._environment_yaml
+
+    @property
+    def _should_preassemble_env(self):
+        """Check for local pip requirements in environment.yaml
+
+        If there are any local references, e.g. `-e .`,
+        stage the whole repo prior to installation.
+        """
+        dependencies = self.environment_yaml.get("dependencies", [])
+        pip_requirements = None
+        for dep in dependencies:
+            if isinstance(dep, dict) and dep.get("pip"):
+                pip_requirements = dep["pip"]
+        if isinstance(pip_requirements, list):
+            for line in pip_requirements:
+                if is_local_pip_requirement(line):
+                    return False
+        return True
+
     @property
     def python_version(self):
         """Detect the Python version for a given `environment.yml`
@@ -135,31 +180,17 @@ def python_version(self):
         or a Falsy empty string '' if not found.
 
         """
-        environment_yml = self.binder_path("environment.yml")
-        if not os.path.exists(environment_yml):
-            return ""
-
         if not hasattr(self, "_python_version"):
             py_version = None
-            with open(environment_yml) as f:
-                env = YAML().load(f)
-                # check if the env file is empty, if so instantiate an empty dictionary.
-                if env is None:
-                    env = {}
-                # check if the env file provided a dick-like thing not a list or other data structure.
-                if not isinstance(env, Mapping):
-                    raise TypeError(
-                        "environment.yml should contain a dictionary. Got %r"
-                        % type(env)
-                    )
-                for dep in env.get("dependencies", []):
-                    if not isinstance(dep, str):
-                        continue
-                    match = PYTHON_REGEX.match(dep)
-                    if not match:
-                        continue
-                    py_version = match.group(1)
-                    break
+            env = self.environment_yaml
+            for dep in env.get("dependencies", []):
+                if not isinstance(dep, str):
+                    continue
+                match = PYTHON_REGEX.match(dep)
+                if not match:
+                    continue
+                py_version = match.group(1)
+                break
 
             # extract major.minor
             if py_version:
@@ -185,19 +216,20 @@ def get_preassemble_script_files(self):
         repo contents change
         """
         assemble_files = super().get_preassemble_script_files()
-        environment_yml = self.binder_path("environment.yml")
-        if os.path.exists(environment_yml):
-            assemble_files[environment_yml] = environment_yml
+        if self._should_preassemble_env:
+            environment_yml = self.binder_path("environment.yml")
+            if os.path.exists(environment_yml):
+                assemble_files[environment_yml] = environment_yml
         return assemble_files
 
-    def get_preassemble_scripts(self):
+    def get_env_scripts(self):
         """Return series of build-steps specific to this source repository.
         """
-        assembly_scripts = []
+        scripts = []
         environment_yml = self.binder_path("environment.yml")
         env_prefix = "${KERNEL_PYTHON_PREFIX}" if self.py2 else "${NB_PYTHON_PREFIX}"
         if os.path.exists(environment_yml):
-            assembly_scripts.append(
+            scripts.append(
                 (
                     "${NB_USER}",
                     r"""
@@ -209,7 +241,19 @@ def get_preassemble_scripts(self):
                     ),
                 )
             )
-        return super().get_preassemble_scripts() + assembly_scripts
+        return scripts
+
+    def get_preassemble_scripts(self):
+        scripts = super().get_preassemble_scripts()
+        if self._should_preassemble_env:
+            scripts.extend(self.get_env_scripts())
+        return scripts
+
+    def get_assemble_scripts(self):
+        scripts = super().get_assemble_scripts()
+        if not self._should_preassemble_env:
+            scripts.extend(self.get_env_scripts())
+        return scripts
 
     def detect(self):
         """Check if current repo should be built with the Conda BuildPack.

repo2docker/buildpacks/python/__init__.py

@@ -2,6 +2,7 @@
 import os
 
 from ..conda import CondaBuildPack
+from ...utils import is_local_pip_requirement
 
 
 class PythonBuildPack(CondaBuildPack):
@@ -34,31 +35,6 @@ def python_version(self):
         self._python_version = py_version
         return self._python_version
 
-    def _is_local_requirement(self, line):
-        """Return whether a line in a requirements.txt file references a local file"""
-        # trim comments and skip empty lines
-        line = line.split("#", 1)[0].strip()
-        if not line:
-            return False
-        if line.startswith(("-r", "-c")):
-            # local -r or -c references break isolation
-            return True
-        # strip off `-e, etc.`
-        if line.startswith("-"):
-            line = line.split(None, 1)[1]
-        if "file://" in line:
-            # file references break isolation
-            return True
-        if "://" in line:
-            # handle git://../local/file
-            path = line.split("://", 1)[1]
-        else:
-            path = line
-        if path.startswith("."):
-            # references a local file
-            return True
-        return False
-
     def _get_pip_scripts(self):
         """Get pip install scripts
 
@@ -112,7 +88,7 @@ def _should_preassemble_pip(self):
                 continue
             with open(requirements_txt) as f:
                 for line in f:
-                    if self._is_local_requirement(line):
+                    if is_local_pip_requirement(line):
                         return False
 
         # didn't find any local references,

repo2docker/utils.py

@@ -431,3 +431,29 @@ def normalize_doi(val):
     (e.g. https://doi.org/10.1234/jshd123)"""
     m = doi_regexp.match(val)
     return m.group(2)
+
+
+def is_local_pip_requirement(line):
+    """Return whether a pip requirement (e.g. in requirements.txt file) references a local file"""
+    # trim comments and skip empty lines
+    line = line.split("#", 1)[0].strip()
+    if not line:
+        return False
+    if line.startswith(("-r", "-c")):
+        # local -r or -c references break isolation
+        return True
+    # strip off `-e, etc.`
+    if line.startswith("-"):
+        line = line.split(None, 1)[1]
+    if "file://" in line:
+        # file references break isolation
+        return True
+    if "://" in line:
+        # handle git://../local/file
+        path = line.split("://", 1)[1]
+    else:
+        path = line
+    if path.startswith("."):
+        # references a local file
+        return True
+    return False