Skip to content

Commit 0998daa

Browse files
hephaexhephaex
committed
Lablup Base kerenl 19.11 for Python 3.6 & CUDA10
1 parent 8ee95f5 commit 0998daa

File tree

1 file changed

+383
-0
lines changed

1 file changed

+383
-0
lines changed

‎commons/Dockerfile.base.19.11-py36-cuda10

Lines changed: 383 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,383 @@
1+
# Labup/Python-xxx Base kernel 19.11
2+
## base Ubuntu 16.04
3+
## Python 3.6.8
4+
## Support TF-hub 0.5.0
5+
## Support OpenCV 4.1.0
6+
## Support Jupyter notebook extension
7+
8+
# base python 3.6
9+
#FROM lablup/common-base:python3.6 as python-binary
10+
11+
FROM nvidia/cuda:10.1-cudnn7-devel-ubuntu18.04
12+
MAINTAINER Mario Cho "[email protected]"
13+
14+
ARG CUDA=10.1
15+
ARG CUDNN=7.6.4.38-1
16+
ENV CUDA_VERSION 10.1.243
17+
ENV CUDA_PKG_VERSION 10-1=$CUDA_VERSION-1
18+
ENV CUDNN_VERSION 7.6.4.38
19+
ENV NCCL_VERSION 2.4.8
20+
21+
SHELL ["/bin/bash", "-c"]
22+
23+
ENV PYTHONUNBUFFERED=1 \
24+
LD_LIBRARY_PATH=/usr/local/cuda/lib64/stubs:/usr/local/cuda/lib64:/usr/local/nvidia/lib64:/usr/local/lib:$LD_LIBRARY_PATH \
25+
PATH="/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin":/usr/local/bin/mecab:$PATH \
26+
LANG=C.UTF-8
27+
28+
RUN apt-get update -y && \
29+
apt-get upgrade -y && \
30+
apt-get install -y --no-install-recommends \
31+
cuda-command-line-tools-${CUDA/./-} \
32+
cuda-core-${CUDA/./-} \
33+
cuda-cufft-${CUDA/./-} \
34+
cuda-curand-${CUDA/./-} \
35+
cuda-cusolver-${CUDA/./-} \
36+
cuda-cusparse-${CUDA/./-} \
37+
cuda-cudart-dev-${CUDA/./-} \
38+
cuda-cufft-dev-${CUDA/./-} \
39+
cuda-curand-dev-${CUDA/./-} \
40+
cuda-cusolver-dev-${CUDA/./-} \
41+
cuda-cusparse-dev-${CUDA/./-} \
42+
ca-certificates \
43+
gcc g++ make \
44+
gfortran \
45+
wget curl \
46+
libssl-dev \
47+
libmpdec2 \
48+
proj-bin libproj-dev \
49+
libgeos-dev libgeos++-dev \
50+
libavfilter-dev \
51+
mime-support \
52+
yasm \
53+
graphviz \
54+
pkg-config \
55+
xz-utils \
56+
wget curl git-core \
57+
vim-tiny bzip2 zip unzip \
58+
python3 python3-pip python3-dev \
59+
&& \
60+
apt-mark hold libnccl2 && \
61+
rm -rf /var/lib/apt/lists/*
62+
63+
# Install CUDA-10.1 + cuDNN 7.6.0
64+
RUN ln -s /usr/local/cuda-10.1 /usr/local/cuda && \
65+
ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so.7.6.0 /usr/local/cuda/lib64/libcudnn.so && \
66+
ldconfig
67+
68+
#COPY --from=python-binary /python.tar.gz /
69+
#RUN cd /; tar xzpf python.tar.gz; rm python.tar.gz; ldconfig
70+
71+
# python alternative support
72+
RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 2
73+
74+
RUN curl https://bootstrap.pypa.io/get-pip.py | python3 && \
75+
python3 -m pip install --no-cache-dir -U setuptools pip
76+
77+
WORKDIR /tmp
78+
COPY nv-tensorrt-repo-ubuntu1804-cuda10.1-trt6.0.1.5-ga-20190913_1-1_amd64.deb /tmp
79+
RUN dpkg -i /tmp/nv-tensorrt-repo-ubuntu1804-cuda10.1-trt6.0.1.5-ga-20190913_1-1_amd64.deb && \
80+
apt-key add /var/nv-tensorrt-repo-cuda10.1-trt6.0.1.5-ga-20190913/7fa2af80.pub && \
81+
apt-get update && \
82+
apt-get install -y --no-install-recommends \
83+
tensorrt \
84+
python3-libnvinfer-dev \
85+
graphsurgeon-tf \
86+
uff-converter-tf \
87+
libnvinfer6 \
88+
libnvinfer-plugin6 \
89+
libnvinfer-plugin-dev \
90+
libnvinfer-dev \
91+
libnvparsers6 \
92+
libnvparsers-dev \
93+
libnvonnxparsers6 \
94+
libnvonnxparsers-dev \
95+
&& \
96+
rm -f nv-tensorrt-repo-ubuntu1804-cuda10.1-trt6.0.1.5-ga-20190913_1-1_amd64.deb
97+
98+
#WORKDIR /tmp
99+
#COPY MLNX_OFED_LINUX-4.4-1.0.0.0-ubuntu18.04-x86_64.tar /tmp
100+
#RUN tar -xvf MLNX_OFED_LINUX-4.4-1.0.0.0-ubuntu18.04-x86_64.tar && \
101+
# cd MLNX_OFED_LINUX-4.4-1.0.0.0-ubuntu18.04-x86_64 && \
102+
# sh ./mlnxofedinstall && \
103+
# rm /tmp/MLNX_OFED_LINUX-4.4-1.0.0.0-ubuntu18.04-x86_64.tar
104+
105+
RUN apt-get update -y && \
106+
apt-get install -y software-properties-common && \
107+
add-apt-repository -y "deb http://security.ubuntu.com/ubuntu xenial-security main" && \
108+
apt-get update -y && \
109+
apt-get install -y --no-install-recommends \
110+
openssh-server \
111+
libopenmpi-dev \
112+
openmpi-bin \
113+
openmpi-common \
114+
openmpi-doc \
115+
binutils \
116+
vim \
117+
libpq-dev \
118+
libjasper-dev \
119+
libtiff-dev \
120+
libjpeg-dev \
121+
libpng-dev \
122+
libavcodec-dev \
123+
libavformat-dev \
124+
libswscale-dev \
125+
libxine2-dev \
126+
libv4l-dev
127+
128+
WORKDIR /tmp
129+
RUN wget https://github.com/Kitware/CMake/releases/download/v3.15.5/cmake-3.15.5.tar.gz && \
130+
tar zxvf cmake-3.15.5.tar.gz && \
131+
cd cmake-3.15.5 && \
132+
bash ./bootstrap && \
133+
make -p${nproc} && \
134+
make install && \
135+
rm -fr /tmp/cmake-3.15.5*
136+
137+
RUN curl -sL https://deb.nodesource.com/setup_10.x | bash - && \
138+
apt-get update -y && \
139+
apt-get install -y nodejs
140+
141+
ENV mecab_dicdir /usr/local/lib/mecab/dic/mecab-ko-dic
142+
WORKDIR /tmp
143+
RUN python3 -m pip install --no-cache-dir Cpython konlpy h5py
144+
145+
RUN apt-get update && \
146+
apt-get install -y \
147+
openjdk-8-jdk \
148+
automake \
149+
&& \
150+
cd /tmp && \
151+
curl -LO https://bitbucket.org/eunjeon/mecab-ko/downloads/mecab-0.996-ko-0.9.2.tar.gz && \
152+
tar zxfv mecab-0.996-ko-0.9.2.tar.gz && \
153+
cd mecab-0.996-ko-0.9.2 && \
154+
./configure && \
155+
make -j$(nproc) && \
156+
make check && \
157+
make install
158+
159+
RUN echo "Install mecab-ko-dic" && \
160+
cd /tmp && \
161+
curl -LO https://bitbucket.org/eunjeon/mecab-ko-dic/downloads/mecab-ko-dic-2.1.1-20180720.tar.gz && \
162+
tar -zxvf mecab-ko-dic-2.1.1-20180720.tar.gz && \
163+
cd mecab-ko-dic-2.1.1-20180720 && \
164+
./autogen.sh && \
165+
./configure && \
166+
make -j$(nproc) && \
167+
sh -c 'echo "dicdir=/usr/local/lib/mecab/dic/mecab-ko-dic" > /usr/local/etc/mecabrc' && \
168+
make install && \
169+
cd /tmp && \
170+
git clone https://bitbucket.org/eunjeon/mecab-python-0.996.git && \
171+
python3 -m pip install /tmp/mecab-python-0.996
172+
173+
RUN ln -s /usr/include/libv4l1-videodev.h /usr/include/linux/videodev.h && \
174+
apt-get install -y \
175+
libgstreamer1.0-dev \
176+
libgstreamer-plugins-base1.0-dev \
177+
libgtk-3-dev \
178+
libtbb-dev \
179+
libatlas-base-dev \
180+
libdc1394-22-dev \
181+
libxvidcore-dev \
182+
libfaac-dev \
183+
libmp3lame-dev \
184+
libtheora-dev \
185+
libvorbis-dev \
186+
libxvidcore-dev \
187+
libopencore-amrnb-dev libopencore-amrwb-dev \
188+
libavresample-dev \
189+
x264 \
190+
libx264-dev \
191+
v4l-utils \
192+
libprotobuf-dev protobuf-compiler \
193+
libgoogle-glog-dev libgflags-dev \
194+
libgphoto2-dev \
195+
libeigen3-dev \
196+
libhdf5-dev \
197+
&& \
198+
apt-get clean && \
199+
rm -rf /var/lib/apt/lists/
200+
201+
202+
WORKDIR /tmp
203+
COPY ./requirements.19.11.txt /tmp
204+
RUN python3 -m pip install --no-cache-dir -r requirements.19.11.txt
205+
206+
# python3 -m pip install --no-cache-dir wheel && \
207+
# python3 -m pip install --no-cache-dir h5py && \
208+
# python3 -m pip install --no-cache-dir Cython && \
209+
# python3 -m pip install --no-cache-dir requests && \
210+
# python3 -m pip install --no-cache-dir numpy==1.14.3 && \
211+
# python3 -m pip install --no-cache-dir scipy==1.1.0 && \
212+
# python3 -m pip install --no-cache-dir cupy && \
213+
# python3 -m pip install --no-cache-dir pyzmq simplejson msgpack-python uvloop && \
214+
# python3 -m pip install --no-cache-dir aiozmq dataclasses tabulate && \
215+
# python3 -m pip install --no-cache-dir namedlist six "python-dateutil>=2" && \
216+
# python3 -m pip install --no-cache-dir versioneer && \
217+
# python3 -m pip install --no-cache-dir pyproj Cartopy && \
218+
# python3 -m pip install --no-cache-dir pandas==0.23.4 && \
219+
# python3 -m pip install --no-cache-dir networkx cvxpy && \
220+
# python3 -m pip install --no-cache-dir scikit-learn==0.20.0 && \
221+
# python3 -m pip install --no-cache-dir lightgbm==2.2.2 && \
222+
# python3 -m pip install --no-cache-dir scikit-image && \
223+
# python3 -m pip install --no-cache-dir pygments && \
224+
# python3 -m pip install --no-cache-dir future
225+
226+
WORKDIR /tmp
227+
228+
# Install Open MPI
229+
RUN mkdir /tmp/openmpi && \
230+
cd /tmp/openmpi && \
231+
wget https://www.open-mpi.org/software/ompi/v3.1/downloads/openmpi-3.1.4.tar.gz && \
232+
tar zxf openmpi-3.1.4.tar.gz && \
233+
cd openmpi-3.1.4 && \
234+
./configure --enable-orterun-prefix-by-default && \
235+
make -j $(nproc) all && \
236+
make install && \
237+
ldconfig && \
238+
rm -rf /tmp/openmpi
239+
240+
241+
ENV OPENCV_VERSION="4.1.0"
242+
RUN wget https://github.com/opencv/opencv/archive/${OPENCV_VERSION}.zip && \
243+
wget -O opencv-contrib.zip https://github.com/opencv/opencv_contrib/archive/${OPENCV_VERSION}.zip && \
244+
unzip ${OPENCV_VERSION}.zip && \
245+
unzip opencv-contrib.zip && \
246+
mkdir opencv-${OPENCV_VERSION}/cmake_binary && \
247+
cd opencv-${OPENCV_VERSION}/cmake_binary && \
248+
cmake \
249+
-DCMAKE_BUILD_TYPE=RELEASE \
250+
-D BUILD_TIFF=ON \
251+
-D BUILD_opencv_java=OFF \
252+
-D WITH_CUDA=ON \
253+
-D CUDA_NVCC_FLAGS=--expt-relaxed-constexpr \
254+
-D CUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda-10.0 \
255+
-D ENABLE_FAST_MATH=1 \
256+
-D CUDA_FAST_MATH=1 \
257+
-D WITH_CUBLAS=1 \
258+
-D WITH_OPENGL=ON \
259+
-D WITH_OPENCL=ON \
260+
-D WITH_IPP=ON \
261+
-D WITH_TBB=ON \
262+
-D WITH_EIGEN=ON \
263+
-D WITH_V4L=ON \
264+
-D BUILD_TESTS=OFF \
265+
-D BUILD_PERF_TESTS=OFF \
266+
-D OPENCV_EXTRA_MODULES_PATH="../../opencv_contrib-4.1.0/modules" \
267+
-D CMAKE_BUILD_TYPE=RELEASE \
268+
-D CMAKE_INSTALL_PREFIX=$(python3 -c "import sys; print(sys.prefix)") \
269+
-D PYTHON_EXECUTABLE=$(which python3) \
270+
-D PYTHON_INCLUDE_DIR=$(python3 -c "from distutils.sysconfig import get_python_inc; print(get_python_inc())") \
271+
-D PYTHON_PACKAGES_PATH=$(python3 -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())") \
272+
.. 2>&1 | tee cmake_messages.txt && \
273+
make -j${nproc} && \
274+
make install && \
275+
cd /tmp && \
276+
rm -fr opencv*
277+
278+
WORKDIR /tmp
279+
RUN wget https://github.com/protocolbuffers/protobuf/releases/download/v3.9.2/protobuf-python-3.9.2.tar.gz && \
280+
tar zxvf protobuf-python-3.9.2.tar.gz && \
281+
cd protobuf-3.9.2 && \
282+
bash ./autogen.sh && \
283+
bash ./configure CXXFLAGS="-fPIC" --prefix=/usr/local --disable-shared && \
284+
make ${nproc} && \
285+
make install && \
286+
rm -fr /tmp/protobuf-python-3.9.2.tar.gz /tmp/proto*
287+
WORKDIR /tmp
288+
RUN git clone --recursive https://github.com/NVIDIA/dali && \
289+
cd dali && \
290+
git checkout release_v0.14 && \
291+
git pull && \
292+
mkdir build && \
293+
cd build && \
294+
cmake .. && \
295+
make -j"$(nproc)" && \
296+
make install && \
297+
python3 -m pip install --no-cache-dir dali/python && \
298+
rm -fr /tmp/dali
299+
300+
# Jupyter notebook extension
301+
RUN mkdir -p /home/work/.jupyter/nbextension
302+
WORKDIR /home/work/.jupyter/nbextension
303+
304+
RUN python3 -m pip install --no-cache-dir opencv-python
305+
306+
RUN python3 -m pip install --no-cache-dir imbalanced-learn && \
307+
python3 -m pip install --no-cache-dir scikit-optimize && \
308+
python3 -m pip install --no-cache-dir bayesian-optimization && \
309+
python3 -m pip install --no-cache-dir scikit-plot && \
310+
python3 -m pip install --no-cache-dir vecstack && \
311+
python3 -m pip install --no-cache-dir category_encoders && \
312+
python3 -m pip install --no-cache-dir scikit-surprise && \
313+
python3 -m pip install --no-cache-dir konlpy && \
314+
python3 -m pip install --no-cache-dir folium && \
315+
python3 -m pip install --no-cache-dir lime && \
316+
python3 -m pip install --no-cache-dir gensim && \
317+
python3 -m pip install --no-cache-dir pyldavis && \
318+
python3 -m pip install --no-cache-dir fbprophet && \
319+
python3 -m pip install --no-cache-dir seaborn && \
320+
python3 -m pip install --no-cache-dir bokeh && \
321+
python3 -m pip install --no-cache-dir plotly && \
322+
python3 -m pip install --no-cache-dir pygal && \
323+
python3 -m pip install --no-cache-dir mlxtend && \
324+
python3 -m pip install --no-cache-dir dtreeviz && \
325+
python3 -m pip install --no-cache-dir pyglet && \
326+
python3 -m pip install --no-cache-dir descartes && \
327+
python3 -m pip install --no-cache-dir geoplotlib && \
328+
python3 -m pip install --no-cache-dir featuretools && \
329+
python3 -m pip install --no-cache-dir graphviz && \
330+
python3 -m pip install --no-cache-dir geopandas && \
331+
python3 -m pip install --no-cache-dir keras_applications && \
332+
python3 -m pip install --no-cache-dir keras_preprocessing && \
333+
python3 -m pip install --no-cache-dir pillow && \
334+
python3 -m pip install --no-cache-dir jupyter && \
335+
python3 -m pip install --no-cache-dir ipython && \
336+
python3 -m pip install --no-cache-dir ipywidgets && \
337+
python3 -m pip install --no-cache-dir ipyparallel && \
338+
python3 -m pip install --no-cache-dir jupyterlab && \
339+
python3 -m pip install --no-cache-dir jupyterthemes && \
340+
python3 -m pip install --no-cache-dir jupyter-js-widgets-nbextension && \
341+
python3 -m pip install --no-cache-dir jupyter_contrib_nbextensions && \
342+
python3 -m pip install --no-cache-dir jupyter_nbextensions_configurator && \
343+
python3 -m pip install --no-cache-dir matplotlib bokeh && \
344+
python3 -m pip install --no-cache-dir tensorflow-gpu && \
345+
python3 -m pip install --no-cache-dir tensorwatch && \
346+
python3 -m pip install --no-cache-dir torchvision && \
347+
python3 -m pip install --no-cache-dir tensorflow-hub && \
348+
python3 -m pip install --no-cache-dir tf2onnx && \
349+
python3 -m pip install --no-cache-dir contextlib2 && \
350+
python3 -m pip install --no-cache-dir pycocotools
351+
352+
353+
354+
# Install Horovod, temporarily using CUDA stubs
355+
RUN ldconfig /usr/local/cuda/targets/x86_64-linux/lib/stubs && \
356+
HOROVOD_GPU_ALLREDUCE=NCCL HOROVOD_WITH_TENSORFLOW=1 HOROVOD_WITH_PYTORCH=1 HOROVOD_WITH_MXNET=1 \
357+
python3 -m pip install --no-cache-dir horovod && \
358+
python3 -m pip install --no-cache-dir mpi4py && \
359+
rm -rf /root/.cache && \
360+
rm -f /tmp/*.whl && \
361+
ldconfig
362+
363+
# Install OpenSSH for MPI to communicate between containers
364+
RUN apt-get install -y --no-install-recommends openssh-client openssh-server && \
365+
mkdir -p /var/run/sshd
366+
367+
# Allow OpenSSH to talk to containers without asking for confirmation
368+
RUN cat /etc/ssh/ssh_config | grep -v StrictHostKeyChecking > /etc/ssh/ssh_config.new && \
369+
echo " StrictHostKeyChecking no" >> /etc/ssh/ssh_config.new && \
370+
mv /etc/ssh/ssh_config.new /etc/ssh/ssh_config
371+
372+
RUN jupyter nbextensions_configurator enable && \
373+
jupyter contrib nbextension install && \
374+
jupyter nbextension enable --py --sys-prefix widgetsnbextension && \
375+
jupyter contrib nbextension install && \
376+
jupyter serverextension enable --py jupyterlab --sys-prefix && \
377+
jupyter labextension install @jupyter-widgets/jupyterlab-manager && \
378+
git clone https://github.com/lambdalisue/jupyter-vim-binding vim_binding && \
379+
jupyter nbextension enable /home/work/.jupyter/nbextension/vim_binding/vim_binding
380+
381+
COPY policy.yml /etc/backend.ai/jail/policy.yml
382+
383+
WORKDIR /home/work

0 commit comments

Comments
 (0)