Skip to content

Commit 977f7e7

Browse files
hephaexhephaex
committed
Merge branch 'master' of github.com:lablup/backend.ai-kernels
2 parents e130e27 + 3685701 commit 977f7e7

File tree

3 files changed

+631
-1
lines changed

3 files changed

+631
-1
lines changed

‎vendor/ngc-pytorch/requirements.22.05.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -261,7 +261,7 @@ sacrebleu==1.3.6
261261
sacred==0.8.1
262262
schema==0.7.1
263263
scikit-image==0.19.0
264-
cikit-optimize==0.9.0
264+
scikit-optimize==0.9.0
265265
scikit-plot==0.3.7
266266
scikit-surprise==1.1.1
267267
scooby==0.5.5

‎vendor/ngc-tensorflow/Dockerfile.22.09-tf2-py3

Lines changed: 255 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,255 @@
1+
FROM nvcr.io/nvidia/tensorflow:22.09-tf2-py3
2+
# NVIDIA DIGITS runs on Python 3.8
3+
4+
ENV DEBIAN_FRONTEND=noninteractive \
5+
MPLBACKEND=Svg \
6+
PIP_IGNORE_INSTALLED=0 \
7+
PYTHONUNBUFFERED=1 \
8+
LD_LIBRARY_PATH="/usr/local/cuda/compat/lib:/usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib:/usr/local/cuda/lib64:/usr/local/nvidia/lib64:/usr/include/x86_64-linux-gnu" \
9+
PATH="/usr/local/nvm/versions/node/v16.15.1/bin:/usr/local/mpi/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/ucx/bin:/opt/tensorrt/bin:/usr/local/src/lightgbm/LightGBM:/usr/local/bin/mecab" \
10+
mecab_dicdir=/usr/local/lib/mecab/dic/mecab-ko-dic \
11+
CPLUS_INCLUDE_PATH=/usr/include/gdal \
12+
C_INCLUDE_PATH=/usr/include/gdal \
13+
LANG=C.UTF-8
14+
15+
RUN curl -sL https://deb.nodesource.com/setup_16.x | bash - && \
16+
ln -s /usr/include/libv4l1-videodev.h /usr/include/linux/videodev.h && \
17+
apt-get update && \
18+
apt-get install -y --no-install-recommends \
19+
automake \
20+
default-libmysqlclient-dev \
21+
ffmpeg \
22+
fonts-nanum \
23+
fonts-nanum-coding \
24+
fonts-nanum-extra \
25+
gfortran \
26+
gir1.2-gtk-3.0 \
27+
htop \
28+
libaio1 \
29+
libasound2-dev \
30+
libatlas-base-dev \
31+
libavformat-dev \
32+
libavresample-dev \
33+
libcairo2-dev \
34+
libdc1394-22-dev \
35+
libeigen3-dev \
36+
libfaac-dev \
37+
libgdal-dev \
38+
libgflags-dev \
39+
libgoogle-glog-dev \
40+
libgirepository1.0-dev \
41+
libgphoto2-dev \
42+
libgstreamer-plugins-base1.0-dev \
43+
libgstreamer1.0-dev \
44+
libgtk-3-dev \
45+
libhdf5-dev \
46+
libmp3lame-dev \
47+
libopenblas-dev \
48+
libopencore-amrnb-dev \
49+
libopencore-amrwb-dev \
50+
libportmidi-dev \
51+
libprotobuf-dev \
52+
libsdl-dev \
53+
libsdl-image1.2-dev \
54+
libsdl-mixer1.2-dev \
55+
libsdl-ttf2.0-dev \
56+
libsm6 \
57+
libswscale-dev \
58+
libtbb-dev \
59+
libtheora-dev \
60+
libvorbis-dev \
61+
libx264-dev \
62+
libxext6 \
63+
libxrender-dev \
64+
libxvidcore-dev \
65+
mercurial \
66+
libncurses-dev \
67+
ncurses-term \
68+
nodejs \
69+
pdsh \
70+
protobuf-compiler \
71+
libsmpeg-dev \
72+
openjdk-8-jdk \
73+
python3-gdal \
74+
v4l-utils \
75+
x264
76+
77+
RUN mkdir -p /opt/oracle && \
78+
cd /opt/oracle && \
79+
wget https://download.oracle.com/otn_software/linux/instantclient/instantclient-basiclite-linuxx64.zip && \
80+
unzip instantclient-basiclite-linuxx64.zip && \
81+
rm -f instantclient-basiclite-linuxx64.zip && \
82+
cd /opt/oracle/instantclient* && \
83+
rm -f *jdbc* *occi* *mysql* *README *jar uidrvci genezi adrci && \
84+
echo /opt/oracle/instantclient* > /etc/ld.so.conf.d/oracle-instantclient.conf && \
85+
ldconfig
86+
87+
RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 2 && \
88+
python3 -m pip install -U "pip < 22" setuptools wheel
89+
90+
# install NLP packages *mecab-ko & khai*
91+
WORKDIR /tmp
92+
RUN curl -LO https://bitbucket.org/eunjeon/mecab-ko/downloads/mecab-0.996-ko-0.9.2.tar.gz && \
93+
tar zxfv mecab-0.996-ko-0.9.2.tar.gz && \
94+
cd mecab-0.996-ko-0.9.2 && \
95+
./configure && \
96+
make -j$(nproc) && \
97+
make check && \
98+
make install && \
99+
echo "Install mecab-ko-dic" && \
100+
cd /tmp && \
101+
ldconfig && \
102+
curl -LO https://bitbucket.org/eunjeon/mecab-ko-dic/downloads/mecab-ko-dic-2.1.1-20180720.tar.gz && \
103+
tar -zxvf mecab-ko-dic-2.1.1-20180720.tar.gz && \
104+
cd mecab-ko-dic-2.1.1-20180720 && \
105+
./autogen.sh && \
106+
./configure && \
107+
make -j$(nproc) && \
108+
sh -c 'echo "dicdir=/usr/local/lib/mecab/dic/mecab-ko-dic" > /usr/local/etc/mecabrc' && \
109+
make install && \
110+
cd /tmp && \
111+
git clone https://bitbucket.org/eunjeon/mecab-python-0.996.git && \
112+
python3 -m pip install /tmp/mecab-python-0.996
113+
114+
# OpenCV
115+
ENV OPENCV_VERSION="4.5.5"
116+
WORKDIR /tmp
117+
RUN wget https://github.com/opencv/opencv/archive/${OPENCV_VERSION}.zip && \
118+
wget -O opencv-contrib.zip https://github.com/opencv/opencv_contrib/archive/${OPENCV_VERSION}.zip && \
119+
unzip ${OPENCV_VERSION}.zip && \
120+
unzip opencv-contrib.zip && \
121+
mkdir opencv-${OPENCV_VERSION}/cmake_binary && \
122+
cd opencv-${OPENCV_VERSION}/cmake_binary && \
123+
cmake \
124+
-DCMAKE_BUILD_TYPE=RELEASE \
125+
-D BUILD_TIFF=ON \
126+
-D BUILD_opencv_java=OFF \
127+
-D WITH_CUDA=ON \
128+
-D CUDA_NVCC_FLAGS=--expt-relaxed-constexpr \
129+
-D CUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda-11.8 \
130+
-D ENABLE_FAST_MATH=1 \
131+
-D CUDA_FAST_MATH=1 \
132+
-D WITH_CUBLAS=1 \
133+
-D WITH_OPENGL=ON \
134+
-D WITH_OPENCL=ON \
135+
-D WITH_IPP=ON \
136+
-D WITH_TBB=ON \
137+
-D WITH_EIGEN=ON \
138+
-D WITH_V4L=ON \
139+
-D BUILD_TESTS=OFF \
140+
-D BUILD_PERF_TESTS=OFF \
141+
-D OPENCV_EXTRA_MODULES_PATH="../../opencv_contrib-4.5.5/modules" \
142+
-D CMAKE_BUILD_TYPE=RELEASE \
143+
-D CMAKE_INSTALL_PREFIX=$(python3 -c "import sys; print(sys.prefix)") \
144+
-D PYTHON_EXECUTABLE=$(which python3) \
145+
-D PYTHON_INCLUDE_DIR=$(python3 -c "from distutils.sysconfig import get_python_inc; print(get_python_inc())") \
146+
-D PYTHON_PACKAGES_PATH=$(python3 -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())") \
147+
.. 2>&1 | tee cmake_messages.txt && \
148+
make -j$(nproc) && \
149+
make install && \
150+
cd /tmp && \
151+
python3 -m pip install --no-cache-dir opencv-python && \
152+
rm -fr opencv*
153+
154+
WORKDIR /tmp
155+
RUN git clone -q --branch=v0.3.20 https://github.com/xianyi/OpenBLAS.git && \
156+
cd OpenBLAS && \
157+
make DYNAMIC_ARCH=1 NO_AFFINITY=1 NUM_THREADS=48 FC=gfortran && \
158+
make install
159+
RUN git clone --recursive https://github.com/bodono/scs-python.git && \
160+
cd /tmp/scs-python && \
161+
python setup.py install --scs --gpu
162+
163+
WORKDIR /tmp
164+
COPY ./requirements.22.09.txt /tmp/requirements.txt
165+
RUN python3 -m pip install --no-cache-dir -r requirements.txt && \
166+
rm -f /tmp/*.whl /tmp/requirements.txt
167+
168+
# install bashtop
169+
WORKDIR /tmp
170+
RUN git clone https://github.com/aristocratos/bashtop.git && \
171+
cd bashtop && \
172+
make install
173+
174+
# install git-lfs
175+
WORKDIR /tmp
176+
RUN curl -sLO https://github.com/git-lfs/git-lfs/releases/download/v3.2.0/git-lfs-linux-amd64-v3.2.0.tar.gz && \
177+
tar -zxf git-lfs-linux-amd64-v3.2.0.tar.gz && \
178+
cd git-lfs-3.2.0 && \
179+
bash install.sh && \
180+
rm -rf /tmp/*
181+
182+
RUN curl -fL https://github.com/cdr/code-server/releases/download/v4.5.1/code-server-4.5.1-linux-amd64.tar.gz | tar -C /usr/local/lib -xz && \
183+
mv /usr/local/lib/code-server-4.5.1-linux-amd64 /usr/local/lib/code-server-4.5.1 && \
184+
ln -s /usr/local/lib/code-server-4.5.1/bin/code-server /usr/local/bin/code-server
185+
186+
# Install Open MPI
187+
RUN mkdir /tmp/openmpi && \
188+
cd /tmp/openmpi && \
189+
wget https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-4.1.4.tar.gz && \
190+
tar zxf openmpi-4.1.4.tar.gz && \
191+
cd openmpi-4.1.4 && \
192+
./configure --enable-orterun-prefix-by-default && \
193+
make -j $(nproc) all && \
194+
make install && \
195+
ldconfig && \
196+
rm -rf /tmp/openmpi*
197+
# Create a wrapper for OpenMPI to allow running as root by default
198+
RUN mv /usr/local/bin/mpirun /usr/local/bin/mpirun.real && \
199+
echo '#!/bin/bash' > /usr/local/bin/mpirun && \
200+
echo 'mpirun.real --allow-run-as-root "$@"' >> /usr/local/bin/mpirun && \
201+
chmod a+x /usr/local/bin/mpirun
202+
203+
# Configure OpenMPI to run good defaults:
204+
RUN echo "btl_tcp_if_exclude = lo,docker0" >> /usr/local/etc/openmpi-mca-params.conf
205+
206+
RUN python3 -m pip install --no-cache-dir \
207+
mpi4py==3.1.3 \
208+
nni==2.8 \
209+
mlflow==1.27.0 \
210+
scikit-nni==0.2.1
211+
212+
# Install Jupyterlab extensions
213+
RUN jupyter nbextensions_configurator enable && \
214+
jupyter contrib nbextension install && \
215+
jupyter nbextension enable --py --sys-prefix widgetsnbextension && \
216+
jupyter serverextension enable --py jupyterlab --sys-prefix && \
217+
jupyter labextension install --no-build @jupyter-widgets/jupyterlab-manager && \
218+
jupyter labextension install --no-build @krassowski/jupyterlab-lsp && \
219+
jupyter serverextension enable --py jupyter_lsp && \
220+
jupyter labextension install --no-build @jupyterlab/toc && \
221+
jupyter nbextension enable execute_time/ExecuteTime && \
222+
jupyter nbextension enable toc2/main && \
223+
jupyter labextension install @jupyterlab/toc && \
224+
jupyter lab build
225+
226+
RUN apt autoclean && \
227+
sed -i 's/source \/usr\/local\/nvm\/nvm.sh//' /etc/bash.bashrc && \
228+
ln -sf /usr/share/terminfo/x/xterm-color /usr/share/terminfo/x/xterm-256color && \
229+
rm -rf /var/lib/apt/lists/* && \
230+
rm -rf /root/.cache && \
231+
rm -rf /tmp/*
232+
233+
# Install ipython kernelspec
234+
RUN /usr/bin/python3 -m ipykernel install --display-name "Python 3.8 (NGC 22.09 / TensorFlow 2.9.1) on Backend.AI" && \
235+
cat /usr/local/share/jupyter/kernels/python3/kernel.json
236+
237+
# Backend.AI specifics
238+
COPY ./service-defs /etc/backend.ai/service-defs
239+
COPY ./runner-scripts/bootstrap.sh runner-scripts/setup_multinode.py /opt/container/
240+
LABEL ai.backend.kernelspec="1" \
241+
ai.backend.envs.corecount="OPENBLAS_NUM_THREADS,OMP_NUM_THREADS,NPROC" \
242+
ai.backend.features="batch query uid-match user-input" \
243+
ai.backend.base-distro="ubuntu16.04" \
244+
ai.backend.accelerators="cuda" \
245+
ai.backend.resource.min.cpu="1" \
246+
ai.backend.resource.min.mem="1g" \
247+
ai.backend.resource.min.cuda.device=0 \
248+
ai.backend.resource.min.cuda.shares=0 \
249+
ai.backend.base-distro="ubuntu16.04" \
250+
ai.backend.runtime-type="python" \
251+
ai.backend.runtime-path="/usr/bin/python" \
252+
ai.backend.service-ports="ipython:pty:3000,jupyter:http:8091,jupyterlab:http:8090,vscode:http:8180,tensorboard:http:6006,mlflow-ui:preopen:5000,nniboard:preopen:8080"
253+
254+
WORKDIR /home/work
255+
# vim: ft=dockerfile

0 commit comments

Comments
 (0)