Skip to content

Commit 02f696a

Browse files
hephaexhephaex
committed
update ML packages
1 parent cf5a7eb commit 02f696a

File tree

1 file changed

+338
-4
lines changed

1 file changed

+338
-4
lines changed

‎python-pytorch/Dockerfile.1.4-py36-cuda10

Lines changed: 338 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,349 @@
1-
# Lablup/Python-PyTorch 1.4 Python 3.6, CUDA10.0
1+
FROM nvidia/cuda:10.0-devel-ubuntu18.04
22

3-
FROM lablup/common-base:19.09-py36-cuda10
4-
MAINTAINER Mario Cho "[email protected]"
3+
# TensorFlow version is tightly coupled to CUDA and cuDNN so it should be selected carefully
4+
ENV PYTORCH_VERSION=1.4.0
5+
ENV TORCHVISION_VERSION=0.5.0
6+
ENV TORCHAUDIO_VERSION=0.4.0
7+
ENV TORCHTEXT_VERSION=0.5.0
8+
ENV TENSORBOARDX_VERSION=1.9
9+
ENV CUDNN_VERSION=7.6.0.64-1+cuda10.0
10+
ENV NCCL_VERSION=2.4.8-1+cuda10.0
11+
ENV DEBIAN_FRONTEND=noninteractive
12+
ENV mecab_dicdir /usr/local/lib/mecab/dic/mecab-ko-dic
13+
14+
# Python 2.7 or 3.6 is supported by Ubuntu Bionic out of the box
15+
ARG python=3.6
16+
ENV PYTHON_VERSION=${python}
517

618
ENV PYTHONUNBUFFERED=1 \
719
LD_LIBRARY_PATH="/usr/local/cuda/lib64:/usr/local/nvidia/lib64" \
8-
PATH="/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \
20+
PATH="/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin/mecab" \
921
LANG=C.UTF-8
1022

23+
# Set default shell to /bin/bash
24+
SHELL ["/bin/bash", "-cu"]
25+
26+
RUN apt-get update -y && \
27+
apt-get install -y --no-install-recommends software-properties-common && \
28+
add-apt-repository -y "deb http://security.ubuntu.com/ubuntu xenial-security main" && \
29+
apt-get update -y && \
30+
apt-get install -y --allow-downgrades --allow-change-held-packages --no-install-recommends \
31+
build-essential \
32+
ca-certificates \
33+
gcc g++ make \
34+
gfortran \
35+
dkms \
36+
git \
37+
curl \
38+
vim \
39+
yasm \
40+
wget zip unzip \
41+
ca-certificates \
42+
software-properties-common \
43+
openssh-client openssh-server \
44+
libssl-dev \
45+
libmpdec2 \
46+
pdsh curl sudo net-tools \
47+
iputils-ping \
48+
libcudnn7=${CUDNN_VERSION} \
49+
libnccl2=${NCCL_VERSION} \
50+
libnccl-dev=${NCCL_VERSION} \
51+
libjpeg-dev \
52+
libpng-dev \
53+
libsm6 \
54+
libxext6 \
55+
libxrender-dev \
56+
libcairo2-dev libgirepository1.0-dev pkg-config gir1.2-gtk-3.0 \
57+
libgeos-dev libgeos++-dev \
58+
mime-support \
59+
proj-bin libproj-dev libgeos-dev libgeos++-dev graphviz \
60+
python${PYTHON_VERSION} \
61+
python${PYTHON_VERSION}-dev \
62+
libnuma-dev \
63+
libjasper-dev \
64+
libtiff-dev \
65+
libavcodec-dev \
66+
libavformat-dev \
67+
libswscale-dev \
68+
libxine2-dev \
69+
libv4l-dev \
70+
libboost-dev \
71+
libboost-system-dev \
72+
libboost-filesystem-dev \
73+
libfreetype6-dev \
74+
fonts-nanum \
75+
fonts-nanum-coding \
76+
fonts-nanum-extra
77+
78+
RUN curl -sL https://deb.nodesource.com/setup_10.x | bash - && \
79+
apt-get update -y && \
80+
apt-get install -y nodejs
81+
82+
# OFED
83+
ENV STAGE_DIR=/tmp
84+
RUN mkdir -p ${STAGE_DIR}
85+
ENV MLNX_OFED_VERSION=4.6-1.0.1.1
86+
RUN cd ${STAGE_DIR} && \
87+
wget -q -O - http://www.mellanox.com/downloads/ofed/MLNX_OFED-${MLNX_OFED_VERSION}/MLNX_OFED_LINUX-${MLNX_OFED_VERSION}-ubuntu18.04-x86_64.tgz | tar xzf -
88+
RUN cd ${STAGE_DIR}/MLNX_OFED_LINUX-${MLNX_OFED_VERSION}-ubuntu18.04-x86_64 && \
89+
./mlnxofedinstall --user-space-only --without-fw-update --all -q && \
90+
cd ${STAGE_DIR} && \
91+
rm -rf ${STAGE_DIR}/MLNX_OFED_LINUX-${MLNX_OFED_VERSION}-ubuntu18.04-x86_64*
92+
93+
# nvidia-peer
94+
RUN mkdir -p ${STAGE_DIR} && \
95+
git clone https://github.com/Mellanox/nv_peer_memory.git ${STAGE_DIR}/nv_peer_memory && \
96+
cd ${STAGE_DIR}/nv_peer_memory && \
97+
./build_module.sh && \
98+
cd ${STAGE_DIR} && \
99+
tar xzf ${STAGE_DIR}/nvidia-peer-memory_1.0.orig.tar.gz && \
100+
cd ${STAGE_DIR}/nvidia-peer-memory-1.0 && \
101+
dpkg-buildpackage -us -uc && \
102+
dpkg -i ${STAGE_DIR}/nvidia-peer-memory_1.0-8_all.deb
103+
104+
# Install CUDA-10.0 + cuDNN 7.6.0
105+
RUN ln -s /usr/local/cuda-10.0 /usr/local/cuda && \
106+
ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so.7.6.0 /usr/local/cuda/lib64/libcudnn.so && \
107+
ldconfig
108+
109+
RUN if [[ "${PYTHON_VERSION}" == "3.6" ]]; then \
110+
apt-get install -y python${PYTHON_VERSION}-distutils python-apt ; \
111+
fi
112+
11113
RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 2
12114

115+
WORKDIR /tmp
116+
RUN curl https://bootstrap.pypa.io/get-pip.py | python3 && \
117+
python3 -m pip install --no-cache-dir -U setuptools pip
118+
119+
# Install TensorFlow, Keras, PyTorch and MXNet
120+
RUN python3 -m pip install pip --no-cache-dir -Iv \
121+
Cython==0.29.13 \
122+
numpy==1.16.5 \
123+
scipy==1.4.1 \
124+
scikit-image==0.16.2 \
125+
scikit-learn==0.21.3 \
126+
matplotlib==3.1.3 \
127+
tornado==6.0.4 \
128+
pystan==2.19.1.1 \
129+
notebook==6.0.3
130+
131+
RUN python3 -m pip install --no-cache-dir \
132+
https://download.pytorch.org/whl/cu100/torch-${PYTORCH_VERSION}%2Bcu100-cp36-cp36m-linux_x86_64.whl \
133+
https://download.pytorch.org/whl/cu100/torchvision-${TORCHVISION_VERSION}%2Bcu100-cp36-cp36m-linux_x86_64.whl \
134+
https://download.pytorch.org/whl/torchaudio-${TORCHAUDIO_VERSION}-cp36-cp36m-linux_x86_64.whl \
135+
torchtext==${TORCHTEXT_VERSION} && \
136+
python3 -m pip install --no-cache-dir tensorboardX==${TENSORBOARDX_VERSION}
137+
RUN python3 -m pip install --no-cache-dir --extra-index-url \
138+
https://developer.download.nvidia.com/compute/redist/cuda/10.0 \
139+
nvidia-dali
140+
141+
WORKDIR /tmp
142+
COPY ./requirements.20.03.txt /tmp
143+
RUN python3 -m pip install --no-cache-dir -Ir requirements.20.03.txt
144+
145+
RUN python3 -m pip install --no-cache-dir -Iv \
146+
tensorwatch==0.8.10 \
147+
jupyterlab-nvdashboard==0.2.0 \
148+
&& \
149+
python3 -m pip install --no-cache-dir konlpy h5py && \
150+
rm -f /tmp/*.whl /tmp/requirements.20.03.txt
151+
152+
RUN apt-get update && \
153+
apt-get install -y \
154+
openjdk-8-jdk \
155+
automake \
156+
&& \
157+
cd /tmp && \
158+
curl -LO https://bitbucket.org/eunjeon/mecab-ko/downloads/mecab-0.996-ko-0.9.2.tar.gz && \
159+
tar zxfv mecab-0.996-ko-0.9.2.tar.gz && \
160+
cd mecab-0.996-ko-0.9.2 && \
161+
./configure && \
162+
make -j$(nproc) && \
163+
make check && \
164+
make install
165+
166+
RUN echo "Install mecab-ko-dic" && \
167+
cd /tmp && \
168+
ldconfig && \
169+
curl -LO https://bitbucket.org/eunjeon/mecab-ko-dic/downloads/mecab-ko-dic-2.1.1-20180720.tar.gz && \
170+
tar -zxvf mecab-ko-dic-2.1.1-20180720.tar.gz && \
171+
cd mecab-ko-dic-2.1.1-20180720 && \
172+
./autogen.sh && \
173+
./configure && \
174+
make -j$(nproc) && \
175+
sh -c 'echo "dicdir=/usr/local/lib/mecab/dic/mecab-ko-dic" > /usr/local/etc/mecabrc' && \
176+
make install && \
177+
cd /tmp && \
178+
git clone https://bitbucket.org/eunjeon/mecab-python-0.996.git && \
179+
python3 -m pip install /tmp/mecab-python-0.996
180+
181+
RUN ln -s /usr/include/libv4l1-videodev.h /usr/include/linux/videodev.h && \
182+
apt-get install -y \
183+
libgstreamer1.0-dev \
184+
libgstreamer-plugins-base1.0-dev \
185+
libgtk-3-dev \
186+
libtbb-dev \
187+
libatlas-base-dev \
188+
libdc1394-22-dev \
189+
libxvidcore-dev \
190+
libfaac-dev \
191+
libmp3lame-dev \
192+
libtheora-dev \
193+
libvorbis-dev \
194+
libxvidcore-dev \
195+
libopencore-amrnb-dev libopencore-amrwb-dev \
196+
libavresample-dev \
197+
x264 \
198+
libx264-dev \
199+
v4l-utils \
200+
libprotobuf-dev protobuf-compiler \
201+
libgoogle-glog-dev libgflags-dev \
202+
libgphoto2-dev \
203+
libeigen3-dev \
204+
libhdf5-dev \
205+
&& \
206+
apt-get clean && \
207+
rm -rf /var/lib/apt/lists/
208+
209+
RUN wget https://github.com/Kitware/CMake/releases/download/v3.15.3/cmake-3.15.3-Linux-x86_64.sh \
210+
-q -O /tmp/cmake-install.sh && \
211+
chmod u+x /tmp/cmake-install.sh && \
212+
mkdir /usr/bin/cmake && \
213+
/tmp/cmake-install.sh --skip-license --prefix=/usr/bin/cmake && \
214+
rm /tmp/cmake-install.sh
215+
216+
ENV PATH="/usr/bin/cmake/bin:${PATH}"
217+
218+
WORKDIR /tmp
219+
ENV OPENCV_VERSION="4.1.0"
220+
RUN wget https://github.com/opencv/opencv/archive/${OPENCV_VERSION}.zip && \
221+
wget -O opencv-contrib.zip https://github.com/opencv/opencv_contrib/archive/${OPENCV_VERSION}.zip && \
222+
unzip ${OPENCV_VERSION}.zip && \
223+
unzip opencv-contrib.zip && \
224+
mkdir opencv-${OPENCV_VERSION}/cmake_binary && \
225+
cd opencv-${OPENCV_VERSION}/cmake_binary && \
226+
cmake \
227+
-DCMAKE_BUILD_TYPE=RELEASE \
228+
-D BUILD_TIFF=ON \
229+
-D BUILD_opencv_java=OFF \
230+
-D WITH_CUDA=ON \
231+
-D CUDA_NVCC_FLAGS=--expt-relaxed-constexpr \
232+
-D CUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda-10.0 \
233+
-D ENABLE_FAST_MATH=1 \
234+
-D CUDA_FAST_MATH=1 \
235+
-D WITH_CUBLAS=1 \
236+
-D WITH_OPENGL=ON \
237+
-D WITH_OPENCL=ON \
238+
-D WITH_IPP=ON \
239+
-D WITH_TBB=ON \
240+
-D WITH_EIGEN=ON \
241+
-D WITH_V4L=ON \
242+
-D BUILD_TESTS=OFF \
243+
-D BUILD_PERF_TESTS=OFF \
244+
-D OPENCV_EXTRA_MODULES_PATH="../../opencv_contrib-4.1.0/modules" \
245+
-D CMAKE_BUILD_TYPE=RELEASE \
246+
-D CMAKE_INSTALL_PREFIX=$(python3 -c "import sys; print(sys.prefix)") \
247+
-D PYTHON_EXECUTABLE=$(which python3) \
248+
-D PYTHON_INCLUDE_DIR=$(python3 -c "from distutils.sysconfig import get_python_inc; print(get_python_inc())") \
249+
-D PYTHON_PACKAGES_PATH=$(python3 -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())") \
250+
.. 2>&1 | tee cmake_messages.txt && \
251+
make -j$(nproc) && \
252+
make install && \
253+
cd /tmp && \
254+
rm -fr opencv*
255+
256+
RUN python3 -m pip install --no-cache-dir opencv-python && \
257+
rm -rf /root/.cache
258+
259+
# add 19.09.06
260+
RUN git clone --recursive https://github.com/dmlc/xgboost && \
261+
cd xgboost && \
262+
mkdir build && \
263+
cd build && \
264+
cmake .. -DUSE_CUDA=ON -DUSE_NCCL=ON && \
265+
make -j$(nproc) && \
266+
cd ../python-package && \
267+
python3 setup.py install
268+
269+
RUN git clone --recursive https://github.com/Microsoft/LightGBM && \
270+
cd LightGBM && \
271+
mkdir build ; cd build && \
272+
cmake -DUSE_GPU=1 -DOpenCL_LIBRARY=/usr/local/cuda-10.0/lib64/libOpenCL.so -DOpenCL_INCLUDE_DIR=/usr/local/cuda-10.0/include/ .. && \
273+
make -j$(nproc) && \
274+
cd ../python-package && \
275+
python3 setup.py install --precompile
276+
277+
# add 19.09.06
278+
RUN git clone https://github.com/scikit-optimize/scikit-optimize.git && \
279+
cd scikit-optimize && \
280+
python3 -m pip install -r requirements.txt && \
281+
python3 setup.py develop
282+
283+
RUN python3 -m pip install --no-cache-dir imbalanced-learn && \
284+
python3 -m pip install --no-cache-dir bayesian-optimization && \
285+
python3 -m pip install --no-cache-dir scikit-plot && \
286+
python3 -m pip install --no-cache-dir vecstack && \
287+
python3 -m pip install --no-cache-dir category_encoders && \
288+
python3 -m pip install --no-cache-dir scikit-surprise && \
289+
python3 -m pip install --no-cache-dir folium && \
290+
python3 -m pip install --no-cache-dir lime && \
291+
python3 -m pip install --no-cache-dir pyldavis && \
292+
python3 -m pip install --no-cache-dir mlxtend && \
293+
python3 -m pip install --no-cache-dir dtreeviz && \
294+
python3 -m pip install --no-cache-dir pyglet && \
295+
python3 -m pip install --no-cache-dir descartes && \
296+
python3 -m pip install --no-cache-dir pydot && \
297+
python3 -m pip install --no-cache-dir tpot && \
298+
python3 -m pip install --no-cache-dir catboost && \
299+
rm -rf /root/.cache && \
300+
rm -f /tmp/*.whl
301+
302+
# Jupyter notebook extension
303+
RUN mkdir -p /home/work/.jupyter/nbextension
304+
WORKDIR /home/work/.jupyter/nbextension
305+
306+
RUN jupyter nbextensions_configurator enable && \
307+
jupyter contrib nbextension install && \
308+
jupyter nbextension enable --py --sys-prefix widgetsnbextension && \
309+
jupyter contrib nbextension install && \
310+
jupyter serverextension enable --py jupyterlab --sys-prefix && \
311+
jupyter labextension install @jupyter-widgets/jupyterlab-manager && \
312+
git clone https://github.com/lambdalisue/jupyter-vim-binding vim_binding && \
313+
jupyter nbextension enable /home/work/.jupyter/nbextension/vim_binding/vim_binding
314+
315+
# Install Open MPI
316+
RUN mkdir /tmp/openmpi && \
317+
cd /tmp/openmpi && \
318+
wget https://www.open-mpi.org/software/ompi/v4.0/downloads/openmpi-4.0.1.tar.gz && \
319+
tar zxf openmpi-4.0.1.tar.gz && \
320+
cd openmpi-4.0.1 && \
321+
./configure --enable-orterun-prefix-by-default && \
322+
make -j $(nproc) all && \
323+
make install && \
324+
ldconfig && \
325+
rm -rf /tmp/openmpi
326+
327+
# Install Horovod, temporarily using CUDA stubs
328+
RUN ldconfig /usr/local/cuda/targets/x86_64-linux/lib/stubs && \
329+
HOROVOD_GPU_ALLREDUCE=NCCL HOROVOD_GPU_BROADCAST=NCCL \
330+
HOROVOD_WITHOUT_TENSORFLOW=1 HOROVOD_WITH_PYTORCH=1 HOROVOD_WITHOUT_MXNET=1 \
331+
pip install --no-cache-dir horovod==0.19.0 && \
332+
ldconfig
333+
334+
RUN python3 -m pip install --no-cache-dir \
335+
mpi4py==3.0.3 \
336+
nni
337+
338+
# Install OpenSSH for MPI to communicate between containers
339+
RUN apt-get install -y --no-install-recommends openssh-client openssh-server && \
340+
mkdir -p /var/run/sshd
341+
342+
# Allow OpenSSH to talk to containers without asking for confirmation
343+
RUN cat /etc/ssh/ssh_config | grep -v StrictHostKeyChecking > /etc/ssh/ssh_config.new && \
344+
echo " StrictHostKeyChecking no" >> /etc/ssh/ssh_config.new && \
345+
mv /etc/ssh/ssh_config.new /etc/ssh/ssh_config
346+
13347
# Install ipython kernelspec
14348
RUN python3 -m ipykernel install --display-name "PyTorch 1.4 on Python 3.6 (CUDA 10.0)" && \
15349
cat /usr/local/share/jupyter/kernels/python3/kernel.json

0 commit comments

Comments
 (0)