Skip to content

Commit a9c3ac6

Browse files
hephaexhephaex
committed
Tensorflow 1.15 on Python 3.6 & CUDA10.0
1 parent ab25435 commit a9c3ac6

File tree

3 files changed

+638
-0
lines changed

3 files changed

+638
-0
lines changed

‎commons/Dockerfile.base.20.03-py36-cuda10

Lines changed: 324 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,324 @@
1+
FROM nvidia/cuda:10.0-devel-ubuntu18.04
2+
3+
# TensorFlow version is tightly coupled to CUDA and cuDNN so it should be selected carefully
4+
ARG CUDA=10.0
5+
ARG LIBNVINFER=6.0.1-1
6+
ARG LIBNVINFER_MAJOR_VERSION=6
7+
ARG CUDNN=7.6.5.32-1
8+
ENV CUDNN_VERSION=7.6.5.32-1+cuda10.0
9+
ENV NCCL_VERSION=2.4.8-1+cuda10.0
10+
ENV DEBIAN_FRONTEND=noninteractive
11+
ENV mecab_dicdir /usr/local/lib/mecab/dic/mecab-ko-dic
12+
13+
# Python 2.7 or 3.6 is supported by Ubuntu Bionic out of the box
14+
ARG python=3.6
15+
ENV PYTHON_VERSION=${python}
16+
17+
ENV PYTHONUNBUFFERED=1 \
18+
LD_LIBRARY_PATH="/usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64:/usr/local/nvidia/lib64:/usr/include/x86_64-linux-gnu" \
19+
PATH="/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin/mecab" \
20+
LANG=C.UTF-8
21+
22+
# Set default shell to /bin/bash
23+
SHELL ["/bin/bash", "-cu"]
24+
25+
RUN apt-get update -y && \
26+
apt-get install -y --no-install-recommends software-properties-common && \
27+
add-apt-repository -y "deb http://security.ubuntu.com/ubuntu xenial-security main" && \
28+
apt-get update -y && \
29+
apt-get install -y --allow-downgrades --allow-change-held-packages --no-install-recommends \
30+
build-essential \
31+
ca-certificates \
32+
software-properties-common \
33+
g++-4.8 \
34+
git \
35+
gfortran \
36+
dkms \
37+
curl \
38+
yasm \
39+
wget zip unzip \
40+
vim \
41+
pdsh curl net-tools \
42+
vim iputils-ping wget \
43+
libcudnn7=${CUDNN_VERSION} \
44+
libcudnn7-dev=${CUDNN}+cuda${CUDA} \
45+
libnccl2=${NCCL_VERSION} \
46+
libnccl-dev=${NCCL_VERSION} \
47+
libcurl3-dev \
48+
libfreetype6-dev \
49+
libhdf5-serial-dev \
50+
libzmq3-dev \
51+
libjpeg-dev \
52+
libpng-dev \
53+
libsm6 \
54+
libxext6 \
55+
libxrender-dev \
56+
libcairo2-dev libgirepository1.0-dev pkg-config gir1.2-gtk-3.0 \
57+
libgeos-dev libgeos++-dev \
58+
pkg-config \
59+
zlib1g-dev \
60+
mime-support \
61+
proj-bin libproj-dev libgeos-dev libgeos++-dev graphviz \
62+
python${PYTHON_VERSION} \
63+
python${PYTHON_VERSION}-dev \
64+
libnuma-dev \
65+
libjasper-dev \
66+
libtiff-dev \
67+
libavcodec-dev \
68+
libavformat-dev \
69+
libswscale-dev \
70+
libxine2-dev \
71+
libv4l-dev \
72+
libboost-dev \
73+
libboost-system-dev \
74+
libboost-filesystem-dev \
75+
fonts-nanum \
76+
fonts-nanum-coding \
77+
fonts-nanum-extra \
78+
&& \
79+
find /usr/local/cuda-${CUDA}/lib64/ -type f -name 'lib*_static.a' -not -name 'libcudart_static.a' -delete && \
80+
rm /usr/lib/x86_64-linux-gnu/libcudnn_static_v7.a
81+
82+
# Install CUDA-10.0 + cuDNN 7.6.0
83+
RUN ln -s /usr/local/cuda-10.0 /usr/local/cuda && \
84+
ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so.7.6.0 /usr/local/cuda/lib64/libcudnn.so && \
85+
ldconfig
86+
87+
RUN apt-get update && \
88+
apt-get install -y --no-install-recommends libnvinfer${LIBNVINFER_MAJOR_VERSION}=${LIBNVINFER}+cuda${CUDA} \
89+
libnvinfer-dev=${LIBNVINFER}+cuda${CUDA} \
90+
&& apt-get clean \
91+
&& rm -rf /var/lib/apt/lists/*
92+
93+
RUN curl -sL https://deb.nodesource.com/setup_10.x | bash - && \
94+
apt-get update -y && \
95+
apt-get install -y nodejs proj-bin libproj-dev libgeos-dev libgeos++-dev graphviz
96+
97+
RUN apt-get update && \
98+
apt-get install -y --no-install-recommends libnvinfer${LIBNVINFER_MAJOR_VERSION}=${LIBNVINFER}+cuda${CUDA} \
99+
libnvinfer-dev=${LIBNVINFER}+cuda${CUDA} \
100+
&& apt-get clean \
101+
&& rm -rf /var/lib/apt/lists/*
102+
103+
# OFED
104+
ENV STAGE_DIR=/tmp
105+
RUN mkdir -p ${STAGE_DIR}
106+
ENV MLNX_OFED_VERSION=4.6-1.0.1.1
107+
RUN cd ${STAGE_DIR} && \
108+
wget -q -O - http://www.mellanox.com/downloads/ofed/MLNX_OFED-${MLNX_OFED_VERSION}/MLNX_OFED_LINUX-${MLNX_OFED_VERSION}-ubuntu18.04-x86_64.tgz | tar xzf -
109+
RUN cd ${STAGE_DIR}/MLNX_OFED_LINUX-${MLNX_OFED_VERSION}-ubuntu18.04-x86_64 && \
110+
./mlnxofedinstall --user-space-only --without-fw-update --all -q && \
111+
cd ${STAGE_DIR} && \
112+
rm -rf ${STAGE_DIR}/MLNX_OFED_LINUX-${MLNX_OFED_VERSION}-ubuntu18.04-x86_64*
113+
114+
# nvidia-peer
115+
RUN mkdir -p ${STAGE_DIR} && \
116+
git clone https://github.com/Mellanox/nv_peer_memory.git ${STAGE_DIR}/nv_peer_memory && \
117+
cd ${STAGE_DIR}/nv_peer_memory && \
118+
./build_module.sh && \
119+
cd ${STAGE_DIR} && \
120+
tar xzf ${STAGE_DIR}/nvidia-peer-memory_1.0.orig.tar.gz && \
121+
cd ${STAGE_DIR}/nvidia-peer-memory-1.0 && \
122+
apt-get install -y dkms && \
123+
dpkg-buildpackage -us -uc && \
124+
dpkg -i ${STAGE_DIR}/nvidia-peer-memory_1.0-8_all.deb
125+
126+
127+
RUN if [[ "${PYTHON_VERSION}" == "3.6" ]]; then \
128+
apt-get install -y python${PYTHON_VERSION}-distutils; \
129+
fi
130+
131+
RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 2
132+
133+
WORKDIR /tmp
134+
RUN curl https://bootstrap.pypa.io/get-pip.py | python3 && \
135+
python3 -m pip install --no-cache-dir -U setuptools pip
136+
137+
# Install TensorFlow, Keras, PyTorch and MXNet
138+
RUN python3 -m pip install pip --no-cache-dir -Iv \
139+
Cython==0.29.13 \
140+
numpy==1.16.3 \
141+
scipy==1.4.1 \
142+
scikit-image==0.16.2 \
143+
scikit-learn==0.22.2 \
144+
tornado==6.0.4 \
145+
pystan==2.19.1.1 \
146+
pycairo==1.19.0 \
147+
matplotlib==3.1.3
148+
RUN python3 -m pip install pip --no-cache-dir \
149+
Cartopy==0.17.0 \
150+
notebook==6.0.3
151+
152+
WORKDIR /tmp
153+
COPY ./requirements.20.03.cuda10.txt /tmp
154+
RUN python3 -m pip install --no-cache-dir -Ir requirements.20.03.cuda10.txt
155+
156+
# NLP
157+
RUN python3 -m pip install --no-cache-dir konlpy h5py && \
158+
rm -f /tmp/*.whl /tmp/requirements.20.03.cuda10.txt
159+
160+
RUN apt-get update && \
161+
apt-get install -y \
162+
openjdk-8-jdk \
163+
automake \
164+
&& \
165+
cd /tmp && \
166+
curl -LO https://bitbucket.org/eunjeon/mecab-ko/downloads/mecab-0.996-ko-0.9.2.tar.gz && \
167+
tar zxfv mecab-0.996-ko-0.9.2.tar.gz && \
168+
cd mecab-0.996-ko-0.9.2 && \
169+
./configure && \
170+
make -j$(nproc) && \
171+
make check && \
172+
make install
173+
174+
RUN echo "Install mecab-ko-dic" && \
175+
cd /tmp && \
176+
ldconfig && \
177+
curl -LO https://bitbucket.org/eunjeon/mecab-ko-dic/downloads/mecab-ko-dic-2.1.1-20180720.tar.gz && \
178+
tar -zxvf mecab-ko-dic-2.1.1-20180720.tar.gz && \
179+
cd mecab-ko-dic-2.1.1-20180720 && \
180+
./autogen.sh && \
181+
./configure && \
182+
make -j$(nproc) && \
183+
sh -c 'echo "dicdir=/usr/local/lib/mecab/dic/mecab-ko-dic" > /usr/local/etc/mecabrc' && \
184+
make install && \
185+
cd /tmp && \
186+
git clone https://bitbucket.org/eunjeon/mecab-python-0.996.git && \
187+
python3 -m pip install /tmp/mecab-python-0.996
188+
189+
# OpenCV
190+
RUN ln -s /usr/include/libv4l1-videodev.h /usr/include/linux/videodev.h && \
191+
apt-get install -y \
192+
libgstreamer1.0-dev \
193+
libgstreamer-plugins-base1.0-dev \
194+
libgtk-3-dev \
195+
libtbb-dev \
196+
libatlas-base-dev \
197+
libdc1394-22-dev \
198+
libxvidcore-dev \
199+
libfaac-dev \
200+
libmp3lame-dev \
201+
libtheora-dev \
202+
libvorbis-dev \
203+
libxvidcore-dev \
204+
libopencore-amrnb-dev libopencore-amrwb-dev \
205+
libavresample-dev \
206+
x264 \
207+
libx264-dev \
208+
v4l-utils \
209+
libprotobuf-dev protobuf-compiler \
210+
libgoogle-glog-dev libgflags-dev \
211+
libgphoto2-dev \
212+
libeigen3-dev \
213+
libhdf5-dev \
214+
&& \
215+
apt-get clean && \
216+
rm -rf /var/lib/apt/lists/
217+
218+
RUN wget https://github.com/Kitware/CMake/releases/download/v3.15.3/cmake-3.15.3-Linux-x86_64.sh \
219+
-q -O /tmp/cmake-install.sh && \
220+
chmod u+x /tmp/cmake-install.sh && \
221+
mkdir /usr/bin/cmake && \
222+
/tmp/cmake-install.sh --skip-license --prefix=/usr/bin/cmake && \
223+
rm /tmp/cmake-install.sh
224+
225+
ENV PATH="/usr/bin/cmake/bin:${PATH}"
226+
227+
WORKDIR /tmp
228+
ENV OPENCV_VERSION="4.1.0"
229+
RUN wget https://github.com/opencv/opencv/archive/${OPENCV_VERSION}.zip && \
230+
wget -O opencv-contrib.zip https://github.com/opencv/opencv_contrib/archive/${OPENCV_VERSION}.zip && \
231+
unzip ${OPENCV_VERSION}.zip && \
232+
unzip opencv-contrib.zip && \
233+
mkdir opencv-${OPENCV_VERSION}/cmake_binary && \
234+
cd opencv-${OPENCV_VERSION}/cmake_binary && \
235+
cmake \
236+
-DCMAKE_BUILD_TYPE=RELEASE \
237+
-D BUILD_TIFF=ON \
238+
-D BUILD_opencv_java=OFF \
239+
-D WITH_CUDA=ON \
240+
-D CUDA_NVCC_FLAGS=--expt-relaxed-constexpr \
241+
-D CUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda-10.0 \
242+
-D ENABLE_FAST_MATH=1 \
243+
-D CUDA_FAST_MATH=1 \
244+
-D WITH_CUBLAS=1 \
245+
-D WITH_OPENGL=ON \
246+
-D WITH_OPENCL=ON \
247+
-D WITH_IPP=ON \
248+
-D WITH_TBB=ON \
249+
-D WITH_EIGEN=ON \
250+
-D WITH_V4L=ON \
251+
-D BUILD_TESTS=OFF \
252+
-D BUILD_PERF_TESTS=OFF \
253+
-D OPENCV_EXTRA_MODULES_PATH="../../opencv_contrib-4.1.0/modules" \
254+
-D CMAKE_BUILD_TYPE=RELEASE \
255+
-D CMAKE_INSTALL_PREFIX=$(python3 -c "import sys; print(sys.prefix)") \
256+
-D PYTHON_EXECUTABLE=$(which python3) \
257+
-D PYTHON_INCLUDE_DIR=$(python3 -c "from distutils.sysconfig import get_python_inc; print(get_python_inc())") \
258+
-D PYTHON_PACKAGES_PATH=$(python3 -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())") \
259+
.. 2>&1 | tee cmake_messages.txt && \
260+
make -j$(nproc) && \
261+
make install && \
262+
cd /tmp && \
263+
python3 -m pip install --no-cache-dir opencv-python && \
264+
rm -fr opencv*
265+
266+
# XGBoost
267+
RUN git clone --recursive https://github.com/dmlc/xgboost && \
268+
cd xgboost && \
269+
mkdir build && \
270+
cd build && \
271+
cmake .. -DUSE_CUDA=ON -DUSE_NCCL=ON && \
272+
make -j$(nproc) && \
273+
cd ../python-package && \
274+
python3 setup.py install
275+
276+
# LightGBM
277+
RUN git clone --recursive https://github.com/Microsoft/LightGBM && \
278+
cd LightGBM && \
279+
mkdir build ; cd build && \
280+
cmake -DUSE_GPU=1 -DOpenCL_LIBRARY=/usr/local/cuda-10.0/lib64/libOpenCL.so -DOpenCL_INCLUDE_DIR=/usr/local/cuda-10.0/include/ .. && \
281+
make -j$(nproc) && \
282+
cd ../python-package && \
283+
python3 setup.py install --precompile
284+
285+
# Install Open MPI
286+
RUN mkdir /tmp/openmpi && \
287+
cd /tmp/openmpi && \
288+
wget https://www.open-mpi.org/software/ompi/v4.0/downloads/openmpi-4.0.1.tar.gz && \
289+
tar zxf openmpi-4.0.1.tar.gz && \
290+
cd openmpi-4.0.1 && \
291+
./configure --enable-orterun-prefix-by-default && \
292+
make -j $(nproc) all && \
293+
make install && \
294+
ldconfig && \
295+
rm -rf /tmp/openmpi*
296+
297+
# Install OpenSSH for MPI to communicate between containers
298+
RUN apt-get update -y ; apt-get install -y --no-install-recommends openssh-client openssh-server && \
299+
mkdir -p /var/run/sshd
300+
301+
# Allow OpenSSH to talk to containers without asking for confirmation
302+
RUN cat /etc/ssh/ssh_config | grep -v StrictHostKeyChecking > /etc/ssh/ssh_config.new && \
303+
echo " StrictHostKeyChecking no" >> /etc/ssh/ssh_config.new && \
304+
mv /etc/ssh/ssh_config.new /etc/ssh/ssh_config
305+
306+
# Jupyter notebook extension
307+
RUN mkdir -p /home/work/.jupyter/nbextension
308+
WORKDIR /home/work/.jupyter/nbextension
309+
310+
RUN jupyter nbextensions_configurator enable && \
311+
jupyter contrib nbextension install && \
312+
jupyter nbextension enable --py --sys-prefix widgetsnbextension && \
313+
jupyter contrib nbextension install && \
314+
jupyter serverextension enable --py jupyterlab --sys-prefix && \
315+
jupyter labextension install @jupyter-widgets/jupyterlab-manager && \
316+
git clone https://github.com/lambdalisue/jupyter-vim-binding vim_binding && \
317+
jupyter nbextension enable /home/work/.jupyter/nbextension/vim_binding/vim_binding
318+
319+
RUN apt autoclean && \
320+
rm -rf /var/lib/apt/lists/* && \
321+
rm -rf /root/.cache && \
322+
rm -rf /tmp/*
323+
324+
WORKDIR /home/work

0 commit comments

Comments
 (0)