Skip to content

Commit 03d673b

Browse files
committed
support separate pip requirements.txt in base env
- require base env to be a lockfile, environment.yml no longer allowed - separate `requirements.py-x.y.txt` can be processed post-install, if present - use this for Python 3.5 env, which has some dependencies only from pip (newer versions than the last py35 build on conda-forge)
1 parent 789a3a2 commit 03d673b

File tree

5 files changed

+136
-111
lines changed

5 files changed

+136
-111
lines changed

repo2docker/buildpacks/conda/__init__.py

Lines changed: 39 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -23,13 +23,17 @@ class CondaBuildPack(BaseImage):
2323
2424
"""
2525

26-
# The kernel environment file, if any.
26+
# The kernel conda environment file, if any.
2727
# As an absolute path within the container.
2828
_kernel_environment_file = ""
29+
# extra pip requirements.txt for the kernel
30+
_kernel_requirements_file = ""
2931

30-
# The notebook server environment file, if any.
32+
# The notebook server environment file.
3133
# As an absolute path within the container.
3234
_nb_environment_file = ""
35+
# extra pip requirements.txt for the notebook env
36+
_nb_requirements_file = ""
3337

3438
def get_build_env(self):
3539
"""Return environment variables to be set.
@@ -38,18 +42,28 @@ def get_build_env(self):
3842
the `NB_PYTHON_PREFIX` to the location of the jupyter binary.
3943
4044
"""
45+
if not self._nb_environment_file:
46+
# get_build_scripts locates requirements/environment files
47+
self.get_build_scripts()
48+
4149
env = super().get_build_env() + [
4250
("CONDA_DIR", "${APP_BASE}/conda"),
4351
("NB_PYTHON_PREFIX", "${CONDA_DIR}/envs/notebook"),
4452
("NB_ENVIRONMENT_FILE", self._nb_environment_file),
4553
]
46-
if self.py2:
54+
if self._nb_requirements_file:
55+
env.append(("NB_REQUIREMENTS_FILE", self._nb_requirements_file))
56+
57+
if self._kernel_environment_file:
58+
# if kernel environment file is separate
4759
env.extend(
4860
[
4961
("KERNEL_PYTHON_PREFIX", "${CONDA_DIR}/envs/kernel"),
5062
("KERNEL_ENVIRONMENT_FILE", self._kernel_environment_file),
5163
]
5264
)
65+
if self._kernel_requirements_file:
66+
env.append(("KERNEL_REQUIREMENTS_FILE", self._kernel_requirements_file))
5367
else:
5468
env.append(("KERNEL_PYTHON_PREFIX", "${NB_PYTHON_PREFIX}"))
5569
return env
@@ -95,7 +109,7 @@ def get_build_scripts(self):
95109
r"""
96110
TIMEFORMAT='time: %3R' \
97111
bash -c 'time /tmp/install-miniforge.bash' && \
98-
rm /tmp/install-miniforge.bash ${NB_ENVIRONMENT_FILE}
112+
rm -rf /tmp/install-miniforge.bash /tmp/env
99113
""",
100114
)
101115
]
@@ -129,24 +143,37 @@ def get_build_script_files(self):
129143
# If no version is specified or no matching X.Y version is found,
130144
# the default base environment is used.
131145
frozen_name = "environment.lock"
146+
pip_frozen_name = "requirements.txt"
132147
if py_version:
133148
if self.py2:
134149
# python 2 goes in a different env
135150
files[
136151
"conda/environment.py-2.7.lock"
137-
] = self._kernel_environment_file = "/tmp/kernel-environment.lock"
152+
] = self._kernel_environment_file = "/tmp/env/kernel-environment.lock"
153+
# additional pip requirements for kernel env
154+
if os.path.exists(os.path.join(HERE, "requirements.py-2.7.txt")):
155+
files[
156+
"conda/requirements.py-2.7.txt"
157+
] = (
158+
self._kernel_requirements_file
159+
) = "/tmp/env/kernel-requirements.txt"
138160
else:
139-
for ext in [".lock", ".frozen.yml"]:
140-
py_frozen_name = f"environment.py-{py_version}{ext}"
141-
if os.path.exists(os.path.join(HERE, py_frozen_name)):
142-
frozen_name = py_frozen_name
143-
break
161+
py_frozen_name = f"environment.py-{py_version}.lock"
162+
if os.path.exists(os.path.join(HERE, py_frozen_name)):
163+
frozen_name = py_frozen_name
164+
pip_frozen_name = f"requirements.py-{py_version}.txt"
144165
if not frozen_name:
145166
self.log.warning(f"No frozen env for {py_version}")
146-
_, frozen_ext = os.path.splitext(frozen_name)
147167
files[
148168
"conda/" + frozen_name
149-
] = self._nb_environment_file = f"/tmp/environment{frozen_ext}"
169+
] = self._nb_environment_file = "/tmp/env/environment.lock"
170+
171+
# add requirements.txt, if present
172+
if os.path.exists(os.path.join(HERE, pip_frozen_name)):
173+
files[
174+
"conda/" + pip_frozen_name
175+
] = self._nb_requirements_file = "/tmp/env/requirements.txt"
176+
150177
files.update(super().get_build_script_files())
151178
return files
152179

repo2docker/buildpacks/conda/environment.py-3.5.frozen.yml

Lines changed: 0 additions & 86 deletions
This file was deleted.
Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
# py3.5 conda packages are no longer being updated as of 12/2018
2+
# this file is hand-maintained now
3+
# and should only receive security fixes, no new features
4+
@EXPLICIT
5+
https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-main.tar.bz2
6+
https://conda.anaconda.org/conda-forge/linux-64/ca-certificates-2018.10.15-ha4d7672_0.tar.bz2
7+
https://repo.anaconda.com/pkgs/main/linux-64/libstdcxx-ng-7.2.0-hdf63c60_3.conda
8+
https://repo.anaconda.com/pkgs/main/linux-64/libgcc-ng-7.2.0-hdf63c60_3.conda
9+
https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.6-h470a237_2.tar.bz2
10+
https://conda.anaconda.org/conda-forge/linux-64/gmp-6.1.2-hfc679d8_0.tar.bz2
11+
https://conda.anaconda.org/conda-forge/linux-64/libffi-3.2.1-hfc679d8_5.tar.bz2
12+
https://conda.anaconda.org/conda-forge/linux-64/libsodium-1.0.16-h470a237_1.tar.bz2
13+
https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.1-hfc679d8_1.tar.bz2
14+
https://conda.anaconda.org/conda-forge/linux-64/openssl-1.0.2p-h470a237_1.tar.bz2
15+
https://conda.anaconda.org/conda-forge/linux-64/xz-5.2.4-h470a237_1.tar.bz2
16+
https://conda.anaconda.org/conda-forge/linux-64/zlib-1.2.11-h470a237_3.tar.bz2
17+
https://conda.anaconda.org/conda-forge/linux-64/pandoc-1.19.2-0.tar.bz2
18+
https://conda.anaconda.org/conda-forge/linux-64/readline-7.0-haf1bffa_1.tar.bz2
19+
https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.9-ha92aebf_0.tar.bz2
20+
https://conda.anaconda.org/conda-forge/linux-64/zeromq-4.2.5-hfc679d8_6.tar.bz2
21+
https://conda.anaconda.org/conda-forge/linux-64/sqlite-3.25.3-hb1c47c0_0.tar.bz2
22+
https://conda.anaconda.org/conda-forge/linux-64/python-3.5.5-h5001a0f_2.tar.bz2
23+
https://conda.anaconda.org/conda-forge/noarch/backcall-0.1.0-py_0.tar.bz2
24+
https://conda.anaconda.org/conda-forge/linux-64/certifi-2018.8.24-py35_1001.tar.bz2
25+
https://conda.anaconda.org/conda-forge/noarch/decorator-4.3.0-py_0.tar.bz2
26+
https://conda.anaconda.org/conda-forge/noarch/defusedxml-0.5.0-py_1.tar.bz2
27+
https://conda.anaconda.org/conda-forge/linux-64/entrypoints-0.2.3-py35_2.tar.bz2
28+
https://conda.anaconda.org/conda-forge/noarch/ipython_genutils-0.2.0-py_1.tar.bz2
29+
https://conda.anaconda.org/conda-forge/linux-64/jsonschema-2.6.0-py35_2.tar.bz2
30+
https://conda.anaconda.org/conda-forge/linux-64/markupsafe-1.0-py35h470a237_1.tar.bz2
31+
https://conda.anaconda.org/conda-forge/linux-64/mistune-0.8.3-py35h470a237_2.tar.bz2
32+
https://conda.anaconda.org/conda-forge/noarch/pandocfilters-1.4.2-py_1.tar.bz2
33+
https://conda.anaconda.org/conda-forge/noarch/parso-0.3.1-py_0.tar.bz2
34+
https://conda.anaconda.org/conda-forge/linux-64/pickleshare-0.7.5-py35_0.tar.bz2
35+
https://conda.anaconda.org/conda-forge/noarch/prometheus_client-0.4.2-py_0.tar.bz2
36+
https://conda.anaconda.org/conda-forge/linux-64/ptyprocess-0.6.0-py35_1000.tar.bz2
37+
https://conda.anaconda.org/conda-forge/linux-64/pyzmq-17.1.2-py35hae99301_0.tar.bz2
38+
https://conda.anaconda.org/conda-forge/noarch/send2trash-1.5.0-py_0.tar.bz2
39+
https://conda.anaconda.org/conda-forge/noarch/simplegeneric-0.8.1-py_1.tar.bz2
40+
https://conda.anaconda.org/conda-forge/linux-64/six-1.11.0-py35_1.tar.bz2
41+
https://conda.anaconda.org/conda-forge/linux-64/testpath-0.3.1-py35_1.tar.bz2
42+
https://conda.anaconda.org/conda-forge/linux-64/tornado-5.1.1-py35h470a237_0.tar.bz2
43+
https://conda.anaconda.org/conda-forge/noarch/wcwidth-0.1.7-py_1.tar.bz2
44+
https://conda.anaconda.org/conda-forge/noarch/webencodings-0.5.1-py_1.tar.bz2
45+
https://conda.anaconda.org/conda-forge/linux-64/jedi-0.12.1-py35_0.tar.bz2
46+
https://conda.anaconda.org/conda-forge/linux-64/pexpect-4.6.0-py35_0.tar.bz2
47+
https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.7.5-py_0.tar.bz2
48+
https://conda.anaconda.org/conda-forge/linux-64/setuptools-40.4.3-py35_0.tar.bz2
49+
https://conda.anaconda.org/conda-forge/linux-64/terminado-0.8.1-py35_1.tar.bz2
50+
https://conda.anaconda.org/conda-forge/linux-64/traitlets-4.3.2-py35_0.tar.bz2
51+
https://conda.anaconda.org/conda-forge/noarch/bleach-3.0.2-py_0.tar.bz2
52+
https://conda.anaconda.org/conda-forge/noarch/jinja2-2.10-py_1.tar.bz2
53+
https://conda.anaconda.org/conda-forge/noarch/jupyter_core-4.4.0-py_0.tar.bz2
54+
https://conda.anaconda.org/conda-forge/noarch/pygments-2.2.0-py_1.tar.bz2
55+
https://conda.anaconda.org/conda-forge/linux-64/wheel-0.32.0-py35_1000.tar.bz2
56+
https://conda.anaconda.org/conda-forge/noarch/jupyter_client-5.2.3-py_1.tar.bz2
57+
https://conda.anaconda.org/conda-forge/noarch/nbformat-4.4.0-py_1.tar.bz2
58+
https://conda.anaconda.org/conda-forge/linux-64/pip-18.0-py35_1001.tar.bz2
59+
https://conda.anaconda.org/conda-forge/noarch/prompt_toolkit-2.0.7-py_0.tar.bz2
60+
https://conda.anaconda.org/conda-forge/linux-64/ipython-7.0.1-py35h24bf2e0_0.tar.bz2
61+
https://conda.anaconda.org/conda-forge/label/broken/noarch/nbconvert-5.4.0-1.tar.bz2
62+
https://conda.anaconda.org/conda-forge/label/broken/noarch/ipykernel-5.1.0-pyh24bf2e0_0.tar.bz2
63+
https://conda.anaconda.org/conda-forge/linux-64/notebook-5.7.0-py35_0.tar.bz2
64+
https://conda.anaconda.org/conda-forge/noarch/jupyterlab_launcher-0.13.1-py_2.tar.bz2
65+
https://conda.anaconda.org/conda-forge/linux-64/widgetsnbextension-3.2.1-py35_1.tar.bz2
66+
https://conda.anaconda.org/conda-forge/linux-64/ipywidgets-7.2.1-py35_1.tar.bz2
67+
https://conda.anaconda.org/conda-forge/linux-64/jupyterlab-0.34.9-py35_0.tar.bz2

repo2docker/buildpacks/conda/install-miniforge.bash

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -47,31 +47,31 @@ time mamba install -y mamba==${MAMBA_VERSION}
4747
echo "installing notebook env:"
4848
cat "${NB_ENVIRONMENT_FILE}"
4949

50-
if [[ "${NB_ENVIRONMENT_FILE: -5}" == ".lock" ]]; then
51-
create="mamba create"
52-
else
53-
create="mamba env create"
54-
fi
55-
56-
time $create -p ${NB_PYTHON_PREFIX} --file "${NB_ENVIRONMENT_FILE}"
50+
time mamba create -p ${NB_PYTHON_PREFIX} --file "${NB_ENVIRONMENT_FILE}"
5751

52+
if [[ ! -z "${NB_REQUIREMENTS_FILE:-}" ]]; then
53+
echo "installing pip requirements"
54+
cat "${NB_REQUIREMENTS_FILE}"
55+
${NB_PYTHON_PREFIX}/bin/python -mpip install --no-cache --no-deps -r "${NB_REQUIREMENTS_FILE}"
56+
fi
5857
# empty conda history file,
5958
# which seems to result in some effective pinning of packages in the initial env,
6059
# which we don't intend.
6160
# this file must not be *removed*, however
6261
echo '' > ${NB_PYTHON_PREFIX}/conda-meta/history
6362

64-
if [[ ! -z "${KERNEL_ENVIRONMENT_FILE:-}" && -f "${KERNEL_ENVIRONMENT_FILE}" ]]; then
63+
if [[ ! -z "${KERNEL_ENVIRONMENT_FILE:-}" ]]; then
6564
# install kernel env and register kernelspec
6665
echo "installing kernel env:"
6766
cat "${KERNEL_ENVIRONMENT_FILE}"
68-
if [[ "${KERNEL_ENVIRONMENT_FILE: -5}" == ".lock" ]]; then
69-
create="mamba create"
70-
else
71-
create="mamba env create"
67+
time mamba create -p ${KERNEL_PYTHON_PREFIX} --file "${KERNEL_ENVIRONMENT_FILE}"
68+
69+
if [[ ! -z "${KERNEL_REQUIREMENTS_FILE:-}" ]]; then
70+
echo "installing pip requirements for kernel"
71+
cat "${KERNEL_REQUIREMENTS_FILE}"
72+
${KERNEL_PYTHON_PREFIX}/bin/python -mpip install --no-cache --no-deps -r "${KERNEL_REQUIREMENTS_FILE}"
7273
fi
7374

74-
time $create -p ${KERNEL_PYTHON_PREFIX} --file "${KERNEL_ENVIRONMENT_FILE}"
7575
${KERNEL_PYTHON_PREFIX}/bin/ipython kernel install --prefix "${NB_PYTHON_PREFIX}"
7676
echo '' > ${KERNEL_PYTHON_PREFIX}/conda-meta/history
7777
mamba list -p ${KERNEL_PYTHON_PREFIX}
Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
# pip-installed packages for Python 3.5
2+
# these should only be packages
3+
# whose versions are too new to have builds available for Python 3.5
4+
alembic==1.0.3
5+
async-generator==1.10
6+
chardet==3.0.4
7+
idna==2.7
8+
jupyterhub==0.9.4
9+
mako==1.0.7
10+
notebook==5.7.8
11+
nteract-on-jupyter==1.9.6
12+
pamela==0.3.0
13+
python-editor==1.0.3
14+
python-oauth2==1.1.0
15+
requests==2.20.1
16+
sqlalchemy==1.2.14
17+
urllib3==1.24.1

0 commit comments

Comments
 (0)