Skip to content

Commit 4c7bd98

Browse files
hephaexhephaex
committed
Base Kernel 20.09
1 parent 091cb34 commit 4c7bd98

File tree

1 file changed

+362
-0
lines changed

1 file changed

+362
-0
lines changed

‎commons/Dockerfile.base.20.09-py36

Lines changed: 362 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,362 @@
1+
FROM ubuntu:18.04
2+
3+
ARG WHL_DIR="/tmp/pip"
4+
5+
# Python 2.7 or 3.6 is supported by Ubuntu Bionic out of the box
6+
ARG python=3.6
7+
ENV PYTHON_VERSION=${python}
8+
9+
ENV DEBIAN_FRONTEND=noninteractive \
10+
MPLBACKEND=Svg \
11+
PYTHONUNBUFFERED=1 \
12+
\
13+
\
14+
PATH="/usr/local/sbin:/usr/bin/cmake/bin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/src/lightgbm/LightGBM:/usr/local/bin/mecab" \
15+
mecab_dicdir=/usr/local/lib/mecab/dic/mecab-ko-dic \
16+
MKL_VERSION=2020.1 \
17+
MKL_BUILD=217 \
18+
LANG=C.UTF-8
19+
20+
# Set default shell to /bin/bash
21+
SHELL ["/bin/bash", "-cu"]
22+
23+
RUN apt-get update -y && \
24+
apt-get install -y --no-install-recommends software-properties-common && \
25+
add-apt-repository -y "deb http://security.ubuntu.com/ubuntu xenial-security main" && \
26+
apt-get update -y && \
27+
apt-get install -y --allow-downgrades --allow-change-held-packages --no-install-recommends \
28+
build-essential \
29+
ca-certificates \
30+
gcc g++ make \
31+
gfortran \
32+
clinfo \
33+
dkms \
34+
git \
35+
curl \
36+
vim \
37+
yasm \
38+
wget zip unzip \
39+
software-properties-common \
40+
openssh-client openssh-server \
41+
libssl-dev \
42+
libmpdec2 \
43+
pdsh curl net-tools \
44+
iputils-ping \
45+
libcurl3-dev \
46+
libfreetype6-dev \
47+
libhdf5-serial-dev \
48+
libzmq3-dev \
49+
libjpeg-dev \
50+
libpng-dev \
51+
libsm6 \
52+
libxext6 \
53+
libxrender-dev \
54+
libcairo2-dev libgirepository1.0-dev pkg-config gir1.2-gtk-3.0 \
55+
libgeos-dev libgeos++-dev \
56+
pkg-config \
57+
gpg-agent \
58+
zlib1g-dev \
59+
mime-support \
60+
proj-bin libproj-dev libgeos-dev libgeos++-dev graphviz \
61+
python${PYTHON_VERSION} \
62+
python${PYTHON_VERSION}-dev \
63+
libxml2-dev \
64+
libxslt1-dev \
65+
libasound2-dev \
66+
libnuma-dev \
67+
libtiff-dev \
68+
libavcodec-dev \
69+
libavformat-dev \
70+
libswscale-dev \
71+
libxine2-dev \
72+
libv4l-dev \
73+
libboost-dev \
74+
libboost-system-dev \
75+
libboost-filesystem-dev \
76+
xvfb \
77+
fonts-nanum \
78+
fonts-nanum-coding \
79+
fonts-nanum-extra \
80+
rsync \
81+
openjdk-8-jdk \
82+
openjdk-8-jre-headless \
83+
openssh-client \
84+
openssh-server \
85+
&& \
86+
apt-get clean && \
87+
rm -rf /var/lib/apt/lists/*
88+
89+
RUN curl -sL https://deb.nodesource.com/setup_10.x | bash - && \
90+
apt-get update -y && \
91+
apt-get install -y nodejs
92+
93+
#install gcc-4.8 and register both 7 and 4.8 with alternatives
94+
#RUN apt-get update && apt-get install -y gcc-4.8 g++-4.8
95+
#RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-4.8 60 --slave /usr/bin/g++ g++ /usr/bin/g++-4.8 && \
96+
# update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 60 --slave /usr/bin/g++ g++ /usr/bin/g++-7
97+
98+
# MKL
99+
WORKDIR /tmp
100+
RUN mkdir -p /opt/intel/lib && \
101+
curl -fsSL https://anaconda.org/intel/mkl-static/${MKL_VERSION}/download/linux-64/mkl-static-${MKL_VERSION}-intel_${MKL_BUILD}.tar.bz2 | tar xjv && \
102+
mv lib/* /opt/intel/lib/ && \
103+
curl -fsSL https://anaconda.org/intel/mkl-include/${MKL_VERSION}/download/linux-64/mkl-include-${MKL_VERSION}-intel_${MKL_BUILD}.tar.bz2 | tar xjv && \
104+
mv include /opt/intel/
105+
106+
# Add OpenCL ICD files for LightGBM
107+
RUN mkdir -p /etc/OpenCL/vendors && \
108+
echo "libnvidia-opencl.so.1" > /etc/OpenCL/vendors/nvidia.icd
109+
110+
# OFED
111+
ENV STAGE_DIR=/tmp
112+
RUN mkdir -p ${STAGE_DIR}
113+
ENV MLNX_OFED_VERSION=4.6-1.0.1.1
114+
RUN cd ${STAGE_DIR} && \
115+
wget -q -O - http://www.mellanox.com/downloads/ofed/MLNX_OFED-${MLNX_OFED_VERSION}/MLNX_OFED_LINUX-${MLNX_OFED_VERSION}-ubuntu18.04-x86_64.tgz | tar xzf -
116+
RUN cd ${STAGE_DIR}/MLNX_OFED_LINUX-${MLNX_OFED_VERSION}-ubuntu18.04-x86_64 && \
117+
./mlnxofedinstall --user-space-only --without-fw-update --all -q && \
118+
cd ${STAGE_DIR} && \
119+
rm -rf ${STAGE_DIR}/MLNX_OFED_LINUX-${MLNX_OFED_VERSION}-ubuntu18.04-x86_64*
120+
121+
RUN if [[ "${PYTHON_VERSION}" == "3.6" ]]; then \
122+
apt-get install -y python${PYTHON_VERSION}-distutils python-apt ; \
123+
fi
124+
125+
RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 2
126+
127+
WORKDIR /tmp
128+
RUN curl https://bootstrap.pypa.io/get-pip.py | python3 && \
129+
python3 -m pip install --no-cache-dir -U setuptools pip
130+
131+
RUN wget https://github.com/Kitware/CMake/releases/download/v3.15.3/cmake-3.15.3-Linux-x86_64.sh \
132+
-q -O /tmp/cmake-install.sh && \
133+
chmod u+x /tmp/cmake-install.sh && \
134+
mkdir /usr/bin/cmake && \
135+
/tmp/cmake-install.sh --skip-license --prefix=/usr/bin/cmake && \
136+
rm /tmp/cmake-install.sh
137+
138+
COPY ./numpy-1.18.5-cp36-cp36m-manylinux2010_x86_64.whl /tmp
139+
COPY ./pandas-1.0.4+4.g29edbab4a-cp36-cp36m-linux_x86_64.whl /tmp
140+
141+
RUN python3 -m pip install --no-cache-dir --upgrade -Iv \
142+
git+https://github.com/haven-jeon/PyKoSpacing.git \
143+
git+https://github.com/ssut/py-hanspell.git \
144+
Cython==0.29.20 \
145+
/tmp/numpy-1.18.5-cp36-cp36m-manylinux2010_x86_64.whl \
146+
/tmp/pandas-1.0.4+4.g29edbab4a-cp36-cp36m-linux_x86_64.whl \
147+
tornado==6.0.4 \
148+
pystan==2.19.1.1 \
149+
pycairo==1.19.0 \
150+
python-language-server[all] \
151+
matplotlib==3.1.1
152+
153+
WORKDIR /tmp
154+
RUN git clone -q --branch=master git://github.com/xianyi/OpenBLAS.git && \
155+
cd OpenBLAS && \
156+
make DYNAMIC_ARCH=1 NO_AFFINITY=1 NUM_THREADS=48 FC=gfortran && \
157+
make install
158+
RUN git clone --recursive https://github.com/bodono/scs-python.git && \
159+
cd /tmp/scs-python && \
160+
python3 setup.py install --scs
161+
162+
RUN python3 -m pip install pip --no-cache-dir \
163+
Cartopy==0.18.0 \
164+
notebook==6.0.3
165+
166+
WORKDIR /tmp
167+
COPY ./requirements.20.08.txt /tmp
168+
RUN python3 -m pip install --no-cache-dir --upgrade -r requirements.20.08.txt && \
169+
python3 -m pip uninstall -y tensorboard tensorboard-plugin-wit tensorflow tensorflow-estimator && \
170+
rm -f /tmp/*.whl /tmp/requirements.20.08.txt
171+
172+
# install NLP packages *mecab-ko & khai*
173+
RUN apt-get update && \
174+
apt-get install -y \
175+
openjdk-8-jdk \
176+
automake \
177+
&& \
178+
cd /tmp && \
179+
curl -LO https://bitbucket.org/eunjeon/mecab-ko/downloads/mecab-0.996-ko-0.9.2.tar.gz && \
180+
tar zxfv mecab-0.996-ko-0.9.2.tar.gz && \
181+
cd mecab-0.996-ko-0.9.2 && \
182+
./configure && \
183+
make -j$(nproc) && \
184+
make check && \
185+
make install
186+
187+
RUN echo "Install mecab-ko-dic" && \
188+
cd /tmp && \
189+
ldconfig && \
190+
curl -LO https://bitbucket.org/eunjeon/mecab-ko-dic/downloads/mecab-ko-dic-2.1.1-20180720.tar.gz && \
191+
tar -zxvf mecab-ko-dic-2.1.1-20180720.tar.gz && \
192+
cd mecab-ko-dic-2.1.1-20180720 && \
193+
./autogen.sh && \
194+
./configure && \
195+
make -j$(nproc) && \
196+
sh -c 'echo "dicdir=/usr/local/lib/mecab/dic/mecab-ko-dic" > /usr/local/etc/mecabrc' && \
197+
make install && \
198+
cd /tmp && \
199+
git clone https://bitbucket.org/eunjeon/mecab-python-0.996.git && \
200+
python3 -m pip install /tmp/mecab-python-0.996
201+
202+
WORKDIR /tmp
203+
RUN git clone https://github.com/kakao/khaiii.git && \
204+
cd khaiii && \
205+
mkdir -p build && \
206+
cd build && \
207+
cmake .. && \
208+
make all && \
209+
make large_resource && \
210+
make install && \
211+
make package_python && \
212+
cd package_python && \
213+
python3 -m pip install .
214+
215+
# OpenCV
216+
RUN ln -s /usr/include/libv4l1-videodev.h /usr/include/linux/videodev.h && \
217+
apt-get install -y \
218+
libgstreamer1.0-dev \
219+
libgstreamer-plugins-base1.0-dev \
220+
libgtk-3-dev \
221+
libtbb-dev \
222+
libatlas-base-dev \
223+
libdc1394-22-dev \
224+
libxvidcore-dev \
225+
libfaac-dev \
226+
libmp3lame-dev \
227+
libtheora-dev \
228+
libvorbis-dev \
229+
libxvidcore-dev \
230+
libopencore-amrnb-dev libopencore-amrwb-dev \
231+
libavresample-dev \
232+
x264 \
233+
libx264-dev \
234+
v4l-utils \
235+
libprotobuf-dev protobuf-compiler \
236+
libgoogle-glog-dev libgflags-dev \
237+
libgphoto2-dev \
238+
libeigen3-dev \
239+
libhdf5-dev \
240+
&& \
241+
apt-get clean && \
242+
rm -rf /var/lib/apt/lists/
243+
244+
WORKDIR /tmp
245+
ENV OPENCV_VERSION="4.4.0"
246+
RUN wget https://github.com/opencv/opencv/archive/${OPENCV_VERSION}.zip && \
247+
wget -O opencv-contrib.zip https://github.com/opencv/opencv_contrib/archive/${OPENCV_VERSION}.zip && \
248+
unzip ${OPENCV_VERSION}.zip && \
249+
unzip opencv-contrib.zip && \
250+
mkdir opencv-${OPENCV_VERSION}/cmake_binary && \
251+
cd opencv-${OPENCV_VERSION}/cmake_binary && \
252+
cmake \
253+
-DCMAKE_BUILD_TYPE=RELEASE \
254+
-D BUILD_TIFF=ON \
255+
-D BUILD_opencv_java=OFF \
256+
-D WITH_CUDA=OFF \
257+
-D ENABLE_FAST_MATH=1 \
258+
-D WITH_OPENGL=ON \
259+
-D WITH_OPENCL=ON \
260+
-D WITH_IPP=ON \
261+
-D WITH_TBB=ON \
262+
-D WITH_EIGEN=ON \
263+
-D WITH_V4L=ON \
264+
-D BUILD_TESTS=OFF \
265+
-D BUILD_PERF_TESTS=OFF \
266+
-D OPENCV_EXTRA_MODULES_PATH="../../opencv_contrib-4.4.0/modules" \
267+
-D CMAKE_BUILD_TYPE=RELEASE \
268+
-D CMAKE_INSTALL_PREFIX=$(python3 -c "import sys; print(sys.prefix)") \
269+
-D PYTHON_EXECUTABLE=$(which python3) \
270+
-D PYTHON_INCLUDE_DIR=$(python3 -c "from distutils.sysconfig import get_python_inc; print(get_python_inc())") \
271+
-D PYTHON_PACKAGES_PATH=$(python3 -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())") \
272+
.. 2>&1 | tee cmake_messages.txt && \
273+
make -j$(nproc) && \
274+
make install && \
275+
cd /tmp && \
276+
python3 -m pip install --no-cache-dir opencv-python && \
277+
rm -fr opencv*
278+
279+
# XGBoost
280+
RUN git clone --recursive https://github.com/dmlc/xgboost && \
281+
cd xgboost && \
282+
mkdir build && \
283+
cd build && \
284+
cmake .. && \
285+
make -j$(nproc) && \
286+
cd ../python-package && \
287+
python3 setup.py install
288+
289+
# LightGBM
290+
291+
RUN cd /usr/local/src && mkdir lightgbm && cd lightgbm && \
292+
git clone --recursive --branch stable --depth 1 https://github.com/microsoft/LightGBM && \
293+
cd LightGBM && mkdir build && cd build && \
294+
cmake -DUSE_GPU=1 -DOpenCL_LIBRARY=/usr/local/cuda/lib64/libOpenCL.so -DOpenCL_INCLUDE_DIR=/usr/local/cuda/include/ .. && \
295+
make OPENCL_HEADERS=/usr/local/cuda-11.0/targets/x86_64-linux/include LIBOPENCL=/usr/local/cuda-11.0/targets/x86_64-linux/lib -j$(nrpoc) && \
296+
cd /usr/local/src/lightgbm/LightGBM/python-package && \
297+
python3 setup.py install --precompile
298+
299+
# Install Open MPI
300+
WORKDIR /root
301+
RUN mkdir /tmp/openmpi && \
302+
cd /tmp/openmpi && \
303+
wget https://www.open-mpi.org/software/ompi/v4.0/downloads/openmpi-4.0.4.tar.gz && \
304+
tar zxf openmpi-4.0.4.tar.gz && \
305+
cd openmpi-4.0.4 && \
306+
./configure --enable-orterun-prefix-by-default && \
307+
make -j $(nproc) all && \
308+
make install && \
309+
ldconfig && \
310+
rm -rf /tmp/openmpi*
311+
# Install OpenSSH for MPI to communicate between containers
312+
RUN apt-get install -y --no-install-recommends openssh-client openssh-server && \
313+
mkdir -p /var/run/sshd
314+
# Allow OpenSSH to talk to containers without asking for confirmation
315+
RUN cat /etc/ssh/ssh_config | grep -v StrictHostKeyChecking > /etc/ssh/ssh_config.new && \
316+
echo " StrictHostKeyChecking no" >> /etc/ssh/ssh_config.new && \
317+
mv /etc/ssh/ssh_config.new /etc/ssh/ssh_config
318+
# Create a wrapper for OpenMPI to allow running as root by default
319+
RUN mv /usr/local/bin/mpirun /usr/local/bin/mpirun.real && \
320+
echo '#!/bin/bash' > /usr/local/bin/mpirun && \
321+
echo 'mpirun.real --allow-run-as-root "$@"' >> /usr/local/bin/mpirun && \
322+
chmod a+x /usr/local/bin/mpirun
323+
# Configure OpenMPI to run good defaults:
324+
RUN echo "btl_tcp_if_exclude = lo,docker0" >> /usr/local/etc/openmpi-mca-params.conf
325+
# Install OpenSSH for MPI to communicate between containers
326+
RUN mkdir -p /var/run/sshd
327+
# Allow OpenSSH to talk to containers without asking for confirmation
328+
RUN cat /etc/ssh/ssh_config | grep -v StrictHostKeyChecking > /etc/ssh/ssh_config.new && \
329+
echo " StrictHostKeyChecking no" >> /etc/ssh/ssh_config.new && \
330+
mv /etc/ssh/ssh_config.new /etc/ssh/ssh_config
331+
332+
# install git-lfs
333+
WORKDIR /tmp
334+
RUN curl -sLO https://github.com/git-lfs/git-lfs/releases/download/v2.11.0/git-lfs-linux-amd64-v2.11.0.tar.gz && \
335+
tar -zxf git-lfs-linux-amd64-v2.11.0.tar.gz && \
336+
bash install.sh && \
337+
rm -rf /tmp/*
338+
339+
COPY ./service-defs /etc/backend.ai/service-defs
340+
RUN curl -fL https://github.com/cdr/code-server/releases/download/v3.4.1/code-server-3.4.1-linux-amd64.tar.gz \
341+
| tar -C /usr/local/lib -xz && \
342+
mv /usr/local/lib/code-server-3.4.1-linux-amd64 /usr/local/lib/code-server-3.4.1 && \
343+
ln -s /usr/local/lib/code-server-3.4.1/bin/code-server /usr/local/bin/code-server
344+
345+
RUN jupyter nbextensions_configurator enable && \
346+
jupyter contrib nbextension install && \
347+
jupyter nbextension enable --py --sys-prefix widgetsnbextension && \
348+
jupyter serverextension enable --py jupyterlab --sys-prefix && \
349+
jupyter labextension install --no-build @jupyter-widgets/jupyterlab-manager && \
350+
jupyter labextension install --no-build @krassowski/jupyterlab-lsp && \
351+
jupyter serverextension enable --py jupyter_lsp && \
352+
jupyter labextension install --no-build @jupyterlab/toc && \
353+
jupyter lab build
354+
355+
RUN apt autoremove -y && \
356+
apt autoclean && \
357+
rm -rf /var/lib/apt/lists/* && \
358+
rm -rf /root/.cache && \
359+
rm -rf /tmp/*
360+
361+
WORKDIR /home/work
362+
# vim: ft=dockerfile

0 commit comments

Comments
 (0)