Skip to content

Commit 0d8be4f

Browse files
hephaexhephaex
committed
Lablup base kernl 20.03 for python 3.6 & CUDA10.1
1 parent cc7c34a commit 0d8be4f

File tree

1 file changed

+322
-0
lines changed

1 file changed

+322
-0
lines changed

‎commons/Dockerfile.base.20.03-py36-cuda10.1

Lines changed: 322 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,322 @@
1+
FROM nvidia/cuda:10.1-devel-ubuntu18.04
2+
3+
# TensorFlow version is tightly coupled to CUDA and cuDNN so it should be selected carefully
4+
ARG CUDA=10.1
5+
ARG LIBNVINFER=6.0.1-1
6+
ARG LIBNVINFER_MAJOR_VERSION=6
7+
ARG CUDNN=7.6.5.32-1
8+
ENV NCCL_VERSION=2.4.8-1+cuda10.1
9+
ENV DEBIAN_FRONTEND=noninteractive
10+
ENV mecab_dicdir /usr/local/lib/mecab/dic/mecab-ko-dic
11+
12+
# Python 2.7 or 3.6 is supported by Ubuntu Bionic out of the box
13+
ARG python=3.6
14+
ENV PYTHON_VERSION=${python}
15+
16+
ENV PYTHONUNBUFFERED=1 \
17+
LD_LIBRARY_PATH="/usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64:/usr/local/nvidia/lib64:/usr/local/cuda/lib64/stubs:/usr/include/x86_64-linux-gnu" \
18+
PATH="/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin/mecab" \
19+
LANG=C.UTF-8
20+
21+
# Set default shell to /bin/bash
22+
SHELL ["/bin/bash", "-cu"]
23+
24+
RUN apt-get update -y && \
25+
apt-get install -y --no-install-recommends software-properties-common && \
26+
add-apt-repository -y "deb http://security.ubuntu.com/ubuntu xenial-security main" && \
27+
apt-get update -y && \
28+
apt-get install -y --allow-downgrades --allow-change-held-packages --no-install-recommends \
29+
build-essential \
30+
ca-certificates \
31+
gcc g++ make \
32+
gfortran \
33+
dkms \
34+
git \
35+
curl \
36+
vim \
37+
yasm \
38+
wget zip unzip \
39+
ca-certificates \
40+
software-properties-common \
41+
openssh-client openssh-server \
42+
libssl-dev \
43+
libmpdec2 \
44+
pdsh curl sudo net-tools \
45+
iputils-ping \
46+
cuda-command-line-tools-${CUDA/./-} \
47+
libcublas10=10.2.1.243-1 \
48+
libcublas-dev=10.2.1.243-1 \
49+
cuda-nvrtc-${CUDA/./-} \
50+
cuda-nvrtc-dev-${CUDA/./-} \
51+
cuda-cudart-dev-${CUDA/./-} \
52+
cuda-cufft-dev-${CUDA/./-} \
53+
cuda-curand-dev-${CUDA/./-} \
54+
cuda-cusolver-dev-${CUDA/./-} \
55+
cuda-cusparse-dev-${CUDA/./-} \
56+
libcudnn7=${CUDNN}+cuda${CUDA} \
57+
libcudnn7-dev=${CUDNN}+cuda${CUDA} \
58+
libnccl2=${NCCL_VERSION} \
59+
libnccl-dev=${NCCL_VERSION} \
60+
libcurl3-dev \
61+
libfreetype6-dev \
62+
libhdf5-serial-dev \
63+
libzmq3-dev \
64+
libjpeg-dev \
65+
libpng-dev \
66+
libsm6 \
67+
libxext6 \
68+
libxrender-dev \
69+
libcairo2-dev libgirepository1.0-dev pkg-config gir1.2-gtk-3.0 \
70+
libgeos-dev libgeos++-dev \
71+
pkg-config \
72+
zlib1g-dev \
73+
mime-support \
74+
proj-bin libproj-dev libgeos-dev libgeos++-dev graphviz \
75+
python${PYTHON_VERSION} \
76+
python${PYTHON_VERSION}-dev \
77+
libnuma-dev \
78+
libjasper-dev \
79+
libtiff-dev \
80+
libavcodec-dev \
81+
libavformat-dev \
82+
libswscale-dev \
83+
libxine2-dev \
84+
libv4l-dev \
85+
libboost-dev \
86+
libboost-system-dev \
87+
libboost-filesystem-dev \
88+
fonts-nanum \
89+
fonts-nanum-coding \
90+
fonts-nanum-extra \
91+
&& \
92+
find /usr/local/cuda-${CUDA}/lib64/ -type f -name 'lib*_static.a' -not -name 'libcudart_static.a' -delete && \
93+
rm /usr/lib/x86_64-linux-gnu/libcudnn_static_v7.a
94+
95+
RUN ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1 \
96+
&& echo "/usr/local/cuda/lib64/stubs" > /etc/ld.so.conf.d/z-cuda-stubs.conf \
97+
&& ldconfig
98+
99+
RUN curl -sL https://deb.nodesource.com/setup_10.x | bash - && \
100+
apt-get update -y && \
101+
apt-get install -y nodejs
102+
103+
RUN apt-get update && \
104+
apt-get install -y --no-install-recommends libnvinfer${LIBNVINFER_MAJOR_VERSION}=${LIBNVINFER}+cuda${CUDA} \
105+
libnvinfer-dev=${LIBNVINFER}+cuda${CUDA} \
106+
libnvinfer-plugin-dev=${LIBNVINFER}+cuda${CUDA} \
107+
libnvinfer-plugin${LIBNVINFER_MAJOR_VERSION}=${LIBNVINFER}+cuda${CUDA} \
108+
&& apt-get clean \
109+
&& rm -rf /var/lib/apt/lists/*
110+
111+
# OFED
112+
ENV STAGE_DIR=/tmp
113+
RUN mkdir -p ${STAGE_DIR}
114+
ENV MLNX_OFED_VERSION=4.6-1.0.1.1
115+
RUN cd ${STAGE_DIR} && \
116+
wget -q -O - http://www.mellanox.com/downloads/ofed/MLNX_OFED-${MLNX_OFED_VERSION}/MLNX_OFED_LINUX-${MLNX_OFED_VERSION}-ubuntu18.04-x86_64.tgz | tar xzf -
117+
RUN cd ${STAGE_DIR}/MLNX_OFED_LINUX-${MLNX_OFED_VERSION}-ubuntu18.04-x86_64 && \
118+
./mlnxofedinstall --user-space-only --without-fw-update --all -q && \
119+
cd ${STAGE_DIR} && \
120+
rm -rf ${STAGE_DIR}/MLNX_OFED_LINUX-${MLNX_OFED_VERSION}-ubuntu18.04-x86_64*
121+
122+
# nvidia-peer
123+
RUN mkdir -p ${STAGE_DIR} && \
124+
git clone https://github.com/Mellanox/nv_peer_memory.git ${STAGE_DIR}/nv_peer_memory && \
125+
cd ${STAGE_DIR}/nv_peer_memory && \
126+
./build_module.sh && \
127+
cd ${STAGE_DIR} && \
128+
tar xzf ${STAGE_DIR}/nvidia-peer-memory_1.0.orig.tar.gz && \
129+
cd ${STAGE_DIR}/nvidia-peer-memory-1.0 && \
130+
dpkg-buildpackage -us -uc && \
131+
dpkg -i ${STAGE_DIR}/nvidia-peer-memory_1.0-8_all.deb
132+
133+
# Install CUDA-10.0 + cuDNN 7.6.0
134+
RUN ln -s /usr/local/cuda-10.0 /usr/local/cuda && \
135+
ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so.7.6.0 /usr/local/cuda/lib64/libcudnn.so && \
136+
ldconfig
137+
138+
RUN if [[ "${PYTHON_VERSION}" == "3.6" ]]; then \
139+
apt-get install -y python${PYTHON_VERSION}-distutils python-apt ; \
140+
fi
141+
142+
RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 2
143+
144+
WORKDIR /tmp
145+
RUN curl https://bootstrap.pypa.io/get-pip.py | python3 && \
146+
python3 -m pip install --no-cache-dir -U setuptools pip
147+
148+
# Install TensorFlow, Keras, PyTorch and MXNet
149+
RUN python3 -m pip install pip --no-cache-dir -Iv \
150+
Cython==0.29.13 \
151+
numpy==1.16.3 \
152+
scipy==1.4.1 \
153+
scikit-image==0.16.2 \
154+
scikit-learn==0.22.2 \
155+
tornado==6.0.4 \
156+
pystan==2.19.1.1 \
157+
pycairo==1.19.0 \
158+
matplotlib==3.1.3
159+
RUN python3 -m pip install pip --no-cache-dir \
160+
Cartopy==0.17.0 \
161+
notebook==6.0.3
162+
163+
WORKDIR /tmp
164+
COPY ./requirements.20.03.txt /tmp
165+
RUN python3 -m pip install --no-cache-dir -Ir requirements.20.03.txt
166+
167+
# NLP
168+
RUN python3 -m pip install --no-cache-dir konlpy h5py && \
169+
rm -f /tmp/*.whl /tmp/requirements.20.03.txt
170+
171+
RUN apt-get update && \
172+
apt-get install -y \
173+
openjdk-8-jdk \
174+
automake \
175+
&& \
176+
cd /tmp && \
177+
curl -LO https://bitbucket.org/eunjeon/mecab-ko/downloads/mecab-0.996-ko-0.9.2.tar.gz && \
178+
tar zxfv mecab-0.996-ko-0.9.2.tar.gz && \
179+
cd mecab-0.996-ko-0.9.2 && \
180+
./configure && \
181+
make -j$(nproc) && \
182+
make check && \
183+
make install
184+
185+
RUN echo "Install mecab-ko-dic" && \
186+
cd /tmp && \
187+
ldconfig && \
188+
curl -LO https://bitbucket.org/eunjeon/mecab-ko-dic/downloads/mecab-ko-dic-2.1.1-20180720.tar.gz && \
189+
tar -zxvf mecab-ko-dic-2.1.1-20180720.tar.gz && \
190+
cd mecab-ko-dic-2.1.1-20180720 && \
191+
./autogen.sh && \
192+
./configure && \
193+
make -j$(nproc) && \
194+
sh -c 'echo "dicdir=/usr/local/lib/mecab/dic/mecab-ko-dic" > /usr/local/etc/mecabrc' && \
195+
make install && \
196+
cd /tmp && \
197+
git clone https://bitbucket.org/eunjeon/mecab-python-0.996.git && \
198+
python3 -m pip install /tmp/mecab-python-0.996
199+
200+
# OpenCV
201+
RUN ln -s /usr/include/libv4l1-videodev.h /usr/include/linux/videodev.h && \
202+
apt-get install -y \
203+
libgstreamer1.0-dev \
204+
libgstreamer-plugins-base1.0-dev \
205+
libgtk-3-dev \
206+
libtbb-dev \
207+
libatlas-base-dev \
208+
libdc1394-22-dev \
209+
libxvidcore-dev \
210+
libfaac-dev \
211+
libmp3lame-dev \
212+
libtheora-dev \
213+
libvorbis-dev \
214+
libxvidcore-dev \
215+
libopencore-amrnb-dev libopencore-amrwb-dev \
216+
libavresample-dev \
217+
x264 \
218+
libx264-dev \
219+
v4l-utils \
220+
libprotobuf-dev protobuf-compiler \
221+
libgoogle-glog-dev libgflags-dev \
222+
libgphoto2-dev \
223+
libeigen3-dev \
224+
libhdf5-dev \
225+
&& \
226+
apt-get clean && \
227+
rm -rf /var/lib/apt/lists/
228+
229+
RUN wget https://github.com/Kitware/CMake/releases/download/v3.15.3/cmake-3.15.3-Linux-x86_64.sh \
230+
-q -O /tmp/cmake-install.sh && \
231+
chmod u+x /tmp/cmake-install.sh && \
232+
mkdir /usr/bin/cmake && \
233+
/tmp/cmake-install.sh --skip-license --prefix=/usr/bin/cmake && \
234+
rm /tmp/cmake-install.sh
235+
236+
ENV PATH="/usr/bin/cmake/bin:${PATH}"
237+
238+
WORKDIR /tmp
239+
ENV OPENCV_VERSION="4.1.0"
240+
RUN wget https://github.com/opencv/opencv/archive/${OPENCV_VERSION}.zip && \
241+
wget -O opencv-contrib.zip https://github.com/opencv/opencv_contrib/archive/${OPENCV_VERSION}.zip && \
242+
unzip ${OPENCV_VERSION}.zip && \
243+
unzip opencv-contrib.zip && \
244+
mkdir opencv-${OPENCV_VERSION}/cmake_binary && \
245+
cd opencv-${OPENCV_VERSION}/cmake_binary && \
246+
cmake \
247+
-DCMAKE_BUILD_TYPE=RELEASE \
248+
-D BUILD_TIFF=ON \
249+
-D BUILD_opencv_java=OFF \
250+
-D WITH_CUDA=ON \
251+
-D CUDA_NVCC_FLAGS=--expt-relaxed-constexpr \
252+
-D CUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda-10.1 \
253+
-D ENABLE_FAST_MATH=1 \
254+
-D CUDA_FAST_MATH=1 \
255+
-D WITH_CUBLAS=1 \
256+
-D WITH_OPENGL=ON \
257+
-D WITH_OPENCL=ON \
258+
-D WITH_IPP=ON \
259+
-D WITH_TBB=ON \
260+
-D WITH_EIGEN=ON \
261+
-D WITH_V4L=ON \
262+
-D BUILD_TESTS=OFF \
263+
-D BUILD_PERF_TESTS=OFF \
264+
-D OPENCV_EXTRA_MODULES_PATH="../../opencv_contrib-4.1.0/modules" \
265+
-D CMAKE_BUILD_TYPE=RELEASE \
266+
-D CMAKE_INSTALL_PREFIX=$(python3 -c "import sys; print(sys.prefix)") \
267+
-D PYTHON_EXECUTABLE=$(which python3) \
268+
-D PYTHON_INCLUDE_DIR=$(python3 -c "from distutils.sysconfig import get_python_inc; print(get_python_inc())") \
269+
-D PYTHON_PACKAGES_PATH=$(python3 -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())") \
270+
.. 2>&1 | tee cmake_messages.txt && \
271+
make -j$(nproc) && \
272+
make install && \
273+
cd /tmp && \
274+
python3 -m pip install --no-cache-dir opencv-python && \
275+
rm -fr opencv*
276+
277+
# XGBoost
278+
RUN git clone --recursive https://github.com/dmlc/xgboost && \
279+
cd xgboost && \
280+
mkdir build && \
281+
cd build && \
282+
cmake .. -DUSE_CUDA=ON -DUSE_NCCL=ON && \
283+
make -j$(nproc) && \
284+
cd ../python-package && \
285+
python3 setup.py install
286+
287+
# LightGBM
288+
RUN git clone --recursive https://github.com/Microsoft/LightGBM && \
289+
cd LightGBM && \
290+
mkdir build ; cd build && \
291+
cmake -DUSE_GPU=1 -DOpenCL_LIBRARY=/usr/local/cuda-10.1/lib64/libOpenCL.so -DOpenCL_INCLUDE_DIR=/usr/local/cuda-10.1/include/ .. && \
292+
make -j$(nproc) && \
293+
cd ../python-package && \
294+
python3 setup.py install --precompile
295+
296+
# Install Open MPI
297+
RUN mkdir /tmp/openmpi && \
298+
cd /tmp/openmpi && \
299+
wget https://www.open-mpi.org/software/ompi/v4.0/downloads/openmpi-4.0.1.tar.gz && \
300+
tar zxf openmpi-4.0.1.tar.gz && \
301+
cd openmpi-4.0.1 && \
302+
./configure --enable-orterun-prefix-by-default && \
303+
make -j $(nproc) all && \
304+
make install && \
305+
ldconfig && \
306+
rm -rf /tmp/openmpi*
307+
308+
# Install OpenSSH for MPI to communicate between containers
309+
RUN apt-get install -y --no-install-recommends openssh-client openssh-server && \
310+
mkdir -p /var/run/sshd
311+
312+
# Allow OpenSSH to talk to containers without asking for confirmation
313+
RUN cat /etc/ssh/ssh_config | grep -v StrictHostKeyChecking > /etc/ssh/ssh_config.new && \
314+
echo " StrictHostKeyChecking no" >> /etc/ssh/ssh_config.new && \
315+
mv /etc/ssh/ssh_config.new /etc/ssh/ssh_config
316+
317+
RUN apt autoclean && \
318+
rm -rf /var/lib/apt/lists/* && \
319+
rm -rf /root/.cache && \
320+
rm -rf /tmp/*
321+
322+
WORKDIR /home/work

0 commit comments

Comments
 (0)