Skip to content

Commit d6e66c8

Browse files
minrkminrk
committed
preassemble python
1 parent 7087808 commit d6e66c8

File tree

1 file changed

+65
-45
lines changed

1 file changed

+65
-45
lines changed

‎repo2docker/buildpacks/python/__init__.py‎

Lines changed: 65 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -34,50 +34,79 @@ def python_version(self):
3434
self._python_version = py_version
3535
return self._python_version
3636

37-
def get_assemble_files(self):
38-
assemble_files = super().get_assemble_files()
39-
for name in ('requirements.txt', 'requirements3.txt'):
40-
requirements_txt = self.binder_path(name)
41-
if os.path.exists(requirements_txt):
42-
assemble_files.append(requirements_txt)
43-
return assemble_files
44-
4537
def _is_local_requirement(self, line):
4638
"""Return whether a line in a requirements.txt file references a local file"""
4739
# trim comments and skip empty lines
48-
line = line.split('#', 1)[0].strip()
40+
line = line.split("#", 1)[0].strip()
4941
if not line:
5042
return False
51-
if line.startswith(('-r', '-c')):
43+
if line.startswith(("-r", "-c")):
5244
# local -r or -c references break isolation
5345
return True
5446
# strip off `-e, etc.`
55-
if line.startswith('-'):
47+
if line.startswith("-"):
5648
line = line.split(None, 1)[1]
57-
if 'file://' in line:
49+
if "file://" in line:
5850
# file references break isolation
5951
return True
60-
if '://' in line:
52+
if "://" in line:
6153
# handle git://../local/file
62-
path = line.split('://', 1)[1]
54+
path = line.split("://", 1)[1]
6355
else:
6456
path = line
65-
if path.startswith('.'):
57+
if path.startswith("."):
6658
# references a local file
6759
return True
6860
return False
6961

62+
def _get_pip_scripts(self):
63+
"""Get pip install scripts
64+
65+
added to preassemble unless local references are found,
66+
in which case this happens in assemble.
67+
"""
68+
# KERNEL_PYTHON_PREFIX is the env with the kernel,
69+
# whether it's distinct from the notebook or the same.
70+
pip = "${KERNEL_PYTHON_PREFIX}/bin/pip"
71+
scripts = []
72+
if self.py2:
73+
# using python 2 kernel,
74+
# requirements3.txt allows installation in the notebook server env
75+
nb_requirements_file = self.binder_path("requirements3.txt")
76+
if os.path.exists(nb_requirements_file):
77+
scripts.append(
78+
(
79+
"${NB_USER}",
80+
# want the $NB_PYHTON_PREFIX environment variable, not for
81+
# Python's string formatting to try and replace this
82+
'${{NB_PYTHON_PREFIX}}/bin/pip install --no-cache-dir -r "{}"'.format(
83+
nb_requirements_file
84+
),
85+
)
86+
)
87+
88+
# install requirements.txt in the kernel env
89+
requirements_file = self.binder_path("requirements.txt")
90+
if os.path.exists(requirements_file):
91+
scripts.append(
92+
(
93+
"${NB_USER}",
94+
'{} install --no-cache-dir -r "{}"'.format(pip, requirements_file),
95+
)
96+
)
97+
return scripts
98+
7099
@property
71-
def assemble_from_subset(self):
100+
def _should_preassemble_pip(self):
72101
"""Peek in requirements.txt to determine if we can assemble from only env files
73102
74103
If there are any local references, e.g. `-e .`,
75104
stage the whole repo prior to installation.
76105
"""
77-
if not os.path.exists('binder') and os.path.exists('setup.py'):
106+
if not os.path.exists("binder") and os.path.exists("setup.py"):
78107
# can't install from subset if we're using setup.py
79108
return False
80-
for name in ('requirements.txt', 'requirements3.txt'):
109+
for name in ("requirements.txt", "requirements3.txt"):
81110
requirements_txt = self.binder_path(name)
82111
if not os.path.exists(requirements_txt):
83112
continue
@@ -90,9 +119,23 @@ def assemble_from_subset(self):
90119
# allow assembly from subset
91120
return True
92121

122+
def get_preassemble_script_files(self):
123+
assemble_files = super().get_preassemble_script_files()
124+
for name in ("requirements.txt", "requirements3.txt"):
125+
requirements_txt = self.binder_path(name)
126+
if os.path.exists(requirements_txt):
127+
assemble_files[requirements_txt] = requirements_txt
128+
return assemble_files
129+
130+
def get_preassemble_scripts(self):
131+
"""Return scripts to run before adding the full repository"""
132+
scripts = super().get_preassemble_scripts()
133+
if self._should_preassemble_pip:
134+
scripts.extend(self._get_pip_scripts())
135+
return scripts
136+
93137
def get_assemble_scripts(self):
94-
"""Return series of build-steps specific to this repository.
95-
"""
138+
"""Return series of build steps that require the full repository"""
96139
# If we have a runtime.txt & that's set to python-2.7,
97140
# requirements.txt will be installed in the *kernel* env
98141
# and requirements3.txt (if it exists)
@@ -102,31 +145,8 @@ def get_assemble_scripts(self):
102145
# KERNEL_PYTHON_PREFIX is the env with the kernel,
103146
# whether it's distinct from the notebook or the same.
104147
pip = "${KERNEL_PYTHON_PREFIX}/bin/pip"
105-
if self.py2:
106-
# using python 2 kernel,
107-
# requirements3.txt allows installation in the notebook server env
108-
nb_requirements_file = self.binder_path("requirements3.txt")
109-
if os.path.exists(nb_requirements_file):
110-
assemble_scripts.append(
111-
(
112-
"${NB_USER}",
113-
# want the $NB_PYHTON_PREFIX environment variable, not for
114-
# Python's string formatting to try and replace this
115-
'${{NB_PYTHON_PREFIX}}/bin/pip install --no-cache-dir -r "{}"'.format(
116-
nb_requirements_file
117-
),
118-
)
119-
)
120-
121-
# install requirements.txt in the kernel env
122-
requirements_file = self.binder_path("requirements.txt")
123-
if os.path.exists(requirements_file):
124-
assemble_scripts.append(
125-
(
126-
"${NB_USER}",
127-
'{} install --no-cache-dir -r "{}"'.format(pip, requirements_file),
128-
)
129-
)
148+
if not self._should_preassemble_pip:
149+
assemble_scripts.extend(self._get_pip_scripts())
130150

131151
# setup.py exists *and* binder dir is not used
132152
if not self.binder_dir and os.path.exists(setup_py):

0 commit comments

Comments
 (0)