Skip to content

Commit 145f046

Browse files
hephaexhephaex
committed
Merge branch '20.12' of https://github.com/lablup/backend.ai-kernels into 20.12
2 parents f4041cb + 4c118d3 commit 145f046

File tree

2 files changed

+261
-8
lines changed

2 files changed

+261
-8
lines changed

‎vendor/ngc-pytorch/Dockerfile.20.12-py3

Lines changed: 253 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,253 @@
1+
FROM nvcr.io/nvidia/pytorch:20.12-py3
2+
# NVIDIA PyTorch with Python 3.6 (CONDA)
3+
4+
5+
6+
ENV DEBIAN_FRONTEND=noninteractive \
7+
MPLBACKEND=Svg \
8+
PYTHONUNBUFFERED=1 \
9+
LD_LIBRARY_PATH="/usr/local/cuda/compat/lib:/usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib:/usr/local/cuda/lib64:/usr/local/nvidia/lib64:/usr/include/x86_64-linux-gnu" \
10+
PATH="/usr/local/nvm/versions/node/v14.8.0/bin:/opt/conda/bin:/opt/cmake-3.14.6-Linux-x86_64/bin/:/usr/local/mpi/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/ucx/bin:/opt/tensorrt/bin:/usr/local/src/lightgbm/LightGBM:/usr/local/bin/mecab" \
11+
mecab_dicdir=/usr/local/lib/mecab/dic/mecab-ko-dic \
12+
CPLUS_INCLUDE_PATH=/usr/include/gdal \
13+
C_INCLUDE_PATH=/usr/include/gdal \
14+
LANG=C.UTF-8
15+
16+
RUN apt-get update && \
17+
apt-get install -y libsm6 libxext6 libxrender-dev mercurial libopenblas-dev libgdal-dev
18+
19+
RUN update-alternatives --install /opt/conda/bin/python python /opt/conda/bin/python3 2
20+
21+
# install NLP packages *mecab-ko & khai*
22+
RUN apt-get update && \
23+
apt-get install -y \
24+
openjdk-8-jdk \
25+
libasound2-dev \
26+
gfortran \
27+
automake \
28+
&& \
29+
cd /tmp && \
30+
curl -LO https://bitbucket.org/eunjeon/mecab-ko/downloads/mecab-0.996-ko-0.9.2.tar.gz && \
31+
tar zxfv mecab-0.996-ko-0.9.2.tar.gz && \
32+
cd mecab-0.996-ko-0.9.2 && \
33+
./configure && \
34+
make -j$(nproc) && \
35+
make check && \
36+
make install
37+
38+
RUN echo "Install mecab-ko-dic" && \
39+
cd /tmp && \
40+
ldconfig && \
41+
curl -LO https://bitbucket.org/eunjeon/mecab-ko-dic/downloads/mecab-ko-dic-2.1.1-20180720.tar.gz && \
42+
tar -zxvf mecab-ko-dic-2.1.1-20180720.tar.gz && \
43+
cd mecab-ko-dic-2.1.1-20180720 && \
44+
./autogen.sh && \
45+
./configure && \
46+
make -j$(nproc) && \
47+
sh -c 'echo "dicdir=/usr/local/lib/mecab/dic/mecab-ko-dic" > /usr/local/etc/mecabrc' && \
48+
make install && \
49+
cd /tmp && \
50+
git clone https://bitbucket.org/eunjeon/mecab-python-0.996.git && \
51+
python3 -m pip install /tmp/mecab-python-0.996
52+
53+
# OpenCV
54+
RUN ln -s /usr/include/libv4l1-videodev.h /usr/include/linux/videodev.h && \
55+
apt-get install -y \
56+
libgstreamer1.0-dev \
57+
libgstreamer-plugins-base1.0-dev \
58+
libgtk-3-dev \
59+
libtbb-dev \
60+
libatlas-base-dev \
61+
libdc1394-22-dev \
62+
libxvidcore-dev \
63+
libfaac-dev \
64+
libmp3lame-dev \
65+
libtheora-dev \
66+
libvorbis-dev \
67+
libxvidcore-dev \
68+
libopencore-amrnb-dev libopencore-amrwb-dev \
69+
libavresample-dev \
70+
x264 \
71+
libx264-dev \
72+
v4l-utils \
73+
libprotobuf-dev protobuf-compiler \
74+
libgoogle-glog-dev libgflags-dev \
75+
libgphoto2-dev \
76+
libeigen3-dev \
77+
libhdf5-dev \
78+
&& \
79+
apt-get clean && \
80+
rm -rf /var/lib/apt/lists/
81+
82+
WORKDIR /tmp
83+
ENV OPENCV_VERSION="4.5.0"
84+
RUN wget https://github.com/opencv/opencv/archive/${OPENCV_VERSION}.zip && \
85+
wget -O opencv-contrib.zip https://github.com/opencv/opencv_contrib/archive/${OPENCV_VERSION}.zip && \
86+
unzip ${OPENCV_VERSION}.zip && \
87+
unzip opencv-contrib.zip && \
88+
mkdir opencv-${OPENCV_VERSION}/cmake_binary && \
89+
cd opencv-${OPENCV_VERSION}/cmake_binary && \
90+
cmake \
91+
-DCMAKE_BUILD_TYPE=RELEASE \
92+
-D BUILD_TIFF=ON \
93+
-D BUILD_opencv_java=OFF \
94+
-D WITH_CUDA=ON \
95+
-D CUDA_NVCC_FLAGS=--expt-relaxed-constexpr \
96+
-D CUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda-11.1 \
97+
-D ENABLE_FAST_MATH=1 \
98+
-D CUDA_FAST_MATH=1 \
99+
-D WITH_CUBLAS=1 \
100+
-D WITH_OPENGL=ON \
101+
-D WITH_OPENCL=ON \
102+
-D WITH_IPP=ON \
103+
-D WITH_TBB=ON \
104+
-D WITH_EIGEN=ON \
105+
-D WITH_V4L=ON \
106+
-D BUILD_TESTS=OFF \
107+
-D BUILD_PERF_TESTS=OFF \
108+
-D OPENCV_EXTRA_MODULES_PATH="../../opencv_contrib-4.5.0/modules" \
109+
-D CMAKE_BUILD_TYPE=RELEASE \
110+
-D CMAKE_INSTALL_PREFIX=$(python3 -c "import sys; print(sys.prefix)") \
111+
-D PYTHON_EXECUTABLE=$(which python3) \
112+
-D PYTHON_INCLUDE_DIR=$(python3 -c "from distutils.sysconfig import get_python_inc; print(get_python_inc())") \
113+
-D PYTHON_PACKAGES_PATH=$(python3 -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())") \
114+
.. 2>&1 | tee cmake_messages.txt && \
115+
make -j$(nproc) && \
116+
make install && \
117+
cd /tmp && \
118+
python3 -m pip install --no-cache-dir opencv-python && \
119+
rm -fr opencv*
120+
WORKDIR /tmp
121+
RUN curl https://bootstrap.pypa.io/get-pip.py | python3 && \
122+
python3 -m pip install --no-cache-dir -U setuptools pip
123+
124+
RUN curl -sL https://deb.nodesource.com/setup_10.x | bash - && \
125+
apt-get update -y && \
126+
apt-get install -y nodejs
127+
128+
WORKDIR /tmp
129+
RUN git clone --recursive https://github.com/bodono/scs-python.git && \
130+
cd /tmp/scs-python && \
131+
python setup.py install --scs --gpu
132+
133+
#COPY ./numpy-1.19.4-cp36-cp36m-manylinux2010_x86_64.whl /tmp
134+
#COPY ./pandas-1.0.4+4.g29edbab4a-cp36-cp36m-linux_x86_64.whl /tmp
135+
RUN /opt/conda/bin/python3 -m pip install --no-cache-dir \
136+
Cython==0.29.21 \
137+
tornado==6.0.4 \
138+
pystan==2.19.1.1 \
139+
pycairo==1.19.0 \
140+
jupyter==1.0.0 \
141+
python-language-server[all] \
142+
matplotlib==3.3.3
143+
ENV SCIPY_VERSION 1.5.4
144+
# Install scipy
145+
RUN cd /tmp && \
146+
git clone --branch=v${SCIPY_VERSION} --depth=1 https://github.com/scipy/scipy.git scipy && \
147+
cd scipy && \
148+
git checkout -b v${SCIPY_VERSION} && \
149+
cp site.cfg.example site.cfg && \
150+
python3 -m pip install -U --no-cache-dir \
151+
# /tmp/numpy-1.19.4-cp36-cp36m-manylinux2010_x86_64.whl \
152+
# /tmp/pandas-1.0.4+4.g29edbab4a-cp36-cp36m-linux_x86_64.whl \
153+
numpy==1.19.4 \
154+
pandas==1.0.4 \
155+
scikit-learn \
156+
hypothesis \
157+
&& \
158+
python3 setup.py install
159+
160+
WORKDIR /tmp
161+
COPY ./requirements.txt /tmp
162+
RUN /opt/conda/bin/python3 -m pip install --no-cache-dir --ignore-installed -r requirements.txt && \
163+
/opt/conda/bin/python3 -m pip install --no-cache-dir tensorflow_model_analysis && \
164+
rm -f /tmp/*.whl /tmp/requirements.txt
165+
166+
# install git-lfs
167+
WORKDIR /tmp
168+
RUN curl -sLO https://github.com/git-lfs/git-lfs/releases/download/v2.12.1/git-lfs-linux-amd64-v2.12.1.tar.gz && \
169+
tar -zxf git-lfs-linux-amd64-v2.12.1.tar.gz && \
170+
bash install.sh && \
171+
rm -rf /tmp/*
172+
173+
COPY ./service-defs /etc/backend.ai/service-defs
174+
RUN curl -fL https://github.com/cdr/code-server/releases/download/v3.7.3/code-server-3.7.3-linux-amd64.tar.gz | tar -C /usr/local/lib -xz && \
175+
mv /usr/local/lib/code-server-3.7.3-linux-amd64 /usr/local/lib/code-server-3.7.3 && \
176+
ln -s /usr/local/lib/code-server-3.7.3/bin/code-server /usr/local/bin/code-server
177+
# Install Open MPI
178+
RUN mkdir /tmp/openmpi && \
179+
cd /tmp/openmpi && \
180+
wget https://www.open-mpi.org/software/ompi/v4.0/downloads/openmpi-4.0.5.tar.gz && \
181+
tar zxf openmpi-4.0.5.tar.gz && \
182+
cd openmpi-4.0.5 && \
183+
./configure --enable-orterun-prefix-by-default && \
184+
make -j $(nproc) all && \
185+
make install && \
186+
ldconfig && \
187+
rm -rf /tmp/openmpi*
188+
# Create a wrapper for OpenMPI to allow running as root by default
189+
RUN mv /usr/local/bin/mpirun /usr/local/bin/mpirun.real && \
190+
echo '#!/bin/bash' > /usr/local/bin/mpirun && \
191+
echo 'mpirun.real --allow-run-as-root "$@"' >> /usr/local/bin/mpirun && \
192+
chmod a+x /usr/local/bin/mpirun
193+
194+
# Configure OpenMPI to run good defaults:
195+
RUN echo "btl_tcp_if_exclude = lo,docker0" >> /usr/local/etc/openmpi-mca-params.conf
196+
197+
# Install Horovod, temporarily using CUDA stubs
198+
RUN ldconfig /usr/local/cuda/targets/x86_64-linux/lib/stubs && \
199+
HOROVOD_CUDA_HOME=$CONDA_PREFIX HOROVOD_GPU_ALLREDUCE=NCCL HOROVOD_GPU_BROADCAST=NCCL HOROVOD_NCCL_LINK=SHARED \
200+
HOROVOD_WITHOUT_TENSORFLOW=1 HOROVOD_WITH_PYTORCH=1 HOROVOD_WITHOUT_MXNET=1 \
201+
pip install --no-cache-dir horovod==0.20.2 && \
202+
ldconfig
203+
204+
RUN python3 -m pip install --no-cache-dir \
205+
mpi4py==3.0.3 \
206+
nni==1.9 \
207+
scikit-nni==0.2.1
208+
209+
RUN jupyter nbextensions_configurator enable && \
210+
jupyter contrib nbextension install && \
211+
jupyter nbextension enable --py --sys-prefix widgetsnbextension && \
212+
jupyter serverextension enable --py jupyterlab --sys-prefix && \
213+
jupyter labextension install --no-build @jupyter-widgets/jupyterlab-manager && \
214+
jupyter labextension install --no-build @krassowski/jupyterlab-lsp && \
215+
jupyter serverextension enable --py jupyter_lsp && \
216+
jupyter labextension install --no-build @jupyterlab/toc && \
217+
jupyter nbextension enable execute_time/ExecuteTime && \
218+
jupyter nbextension enable toc2/main && \
219+
jupyter labextension install @pyviz/jupyterlab_pyviz && \
220+
jupyter labextension install @bokeh/jupyter_bokeh && \
221+
jupyter labextension install --no-build jupyterlab-nvdashboard && \
222+
jupyter lab build
223+
224+
RUN apt autoclean && \
225+
sed -i 's/source \/usr\/local\/nvm\/nvm.sh//' /etc/bash.bashrc && \
226+
python3 -m pip uninstall -y tensorboard-plugin-wit && \
227+
rm -rf /var/lib/apt/lists/* && \
228+
rm -rf /root/.cache && \
229+
rm -rf /tmp/*
230+
231+
RUN /opt/conda/bin/python3 -m ipykernel install \
232+
--prefix=/opt/conda/ \
233+
--display-name "PyTorch 1.8 (NGC 20.12/Python 3.6 Conda) on Backend.AI" && \
234+
cat /opt/conda/share/jupyter/kernels/python3/kernel.json
235+
236+
# Backend.AI specifics
237+
LABEL ai.backend.kernelspec="1" \
238+
ai.backend.envs.corecount="OPENBLAS_NUM_THREADS,OMP_NUM_THREADS,NPROC" \
239+
ai.backend.features="batch query uid-match user-input" \
240+
ai.backend.base-distro="ubuntu16.04" \
241+
ai.backend.accelerators="cuda" \
242+
ai.backend.resource.min.cpu="1" \
243+
ai.backend.resource.min.mem="1g" \
244+
ai.backend.resource.min.cuda.device=1 \
245+
ai.backend.resource.min.cuda.shares=0.1 \
246+
ai.backend.base-distro="ubuntu16.04" \
247+
ai.backend.runtime-type="python" \
248+
ai.backend.runtime-path="/opt/conda/bin/python3" \
249+
ai.backend.service-ports="ipython:pty:3000,jupyter:http:8080,jupyterlab:http:8090,vscode:http:8180,tensorboard:http:6006"
250+
251+
252+
WORKDIR /home/work
253+
# vim: ft=dockerfile

‎vendor/ngc-pytorch/requirements.txt

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ cvxpy==1.0.24
4343
cycler==0.10.0
4444
Cython==0.29.13
4545
dask==2.9.0
46-
dataclasses==0.7
46+
#dataclasses==0.8
4747
deap==1.3.1
4848
decorator==4.4.2
4949
defusedxml==0.6.0
@@ -152,16 +152,16 @@ namedlist==1.7
152152
nbconvert==5.6.1
153153
nbformat==5.0.4
154154
networkx==2.4
155-
ngraph-core==0.26.0
156-
ngraph-onnx==0.24.0
157-
ngraph-tensorflow-bridge==0.18.0
155+
#ngraph-core==0.26.0
156+
#ngraph-onnx==0.24.0
157+
#ngraph-tensorflow-bridge==0.18.0
158158
nltk==3.2.5
159159
numba==0.48.0
160160
numexpr==2.7.0
161161
oauthlib==3.1.0
162162
oauth2client==3.0.0
163-
onnx==1.8.0
164-
#onnx-plaidml==0.6.3
163+
onnx==1.2.1
164+
onnx-plaidml==0.6.3
165165
opt-einsum==3.2.0
166166
osqp==0.5.0
167167
packaging==20.3
@@ -246,7 +246,7 @@ scikit-surprise==1.1.0
246246
seaborn==0.10.0
247247
selenium==3.141.0
248248
Send2Trash==1.5.0
249-
sentencepiece==0.1.82
249+
#sentencepiece==0.1.82
250250
setuptools-git==1.2
251251
Shapely==1.7.0
252252
simpervisor==0.3
@@ -265,7 +265,7 @@ termcolor==1.1.0
265265
terminado==0.8.3
266266
test-generator==0.1.1
267267
testpath==0.4.4
268-
tf2onnx==1.7.2
268+
tf2onnx==0.4.1
269269
toml==0.10.0
270270
toolz==0.10.0
271271
TPOT==0.11.1

0 commit comments

Comments
 (0)