Skip to content

Commit 32b72d9

Browse files
hephaexhephaex
committed
NGC Pytorch 21.02
1 parent 4d9bb5b commit 32b72d9

File tree

2 files changed

+248
-2
lines changed

2 files changed

+248
-2
lines changed

‎vendor/ngc-pytorch/Dockerfile.21.02-py3

Lines changed: 247 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,247 @@
1+
FROM nvcr.io/nvidia/pytorch:21.02-py3
2+
# NVIDIA PyTorch with Python 3.8 (CONDA)
3+
4+
5+
6+
ENV DEBIAN_FRONTEND=noninteractive \
7+
MPLBACKEND=Svg \
8+
PYTHONUNBUFFERED=1 \
9+
LD_LIBRARY_PATH="/usr/local/cuda/compat/lib:/usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib:/usr/local/cuda/lib64:/usr/local/nvidia/lib64:/usr/include/x86_64-linux-gnu" \
10+
PATH="/usr/local/nvm/versions/node/v14.8.0/bin:/opt/conda/bin:/opt/cmake-3.14.6-Linux-x86_64/bin/:/usr/local/mpi/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/ucx/bin:/opt/tensorrt/bin:/usr/local/src/lightgbm/LightGBM:/usr/local/bin/mecab" \
11+
mecab_dicdir=/usr/local/lib/mecab/dic/mecab-ko-dic \
12+
CPLUS_INCLUDE_PATH=/usr/include/gdal \
13+
C_INCLUDE_PATH=/usr/include/gdal \
14+
LANG=C.UTF-8
15+
16+
RUN apt-get update && \
17+
apt-get install -y \
18+
libsm6 libxext6 libxrender-dev mercurial libopenblas-dev libgdal-dev \
19+
openjdk-8-jdk \
20+
libasound2-dev \
21+
gfortran \
22+
libgstreamer1.0-dev \
23+
libgstreamer-plugins-base1.0-dev \
24+
libgtk-3-dev \
25+
libtbb-dev \
26+
libatlas-base-dev \
27+
libdc1394-22-dev \
28+
libxvidcore-dev \
29+
libfaac-dev \
30+
libmp3lame-dev \
31+
libtheora-dev \
32+
libvorbis-dev \
33+
libxvidcore-dev \
34+
libopencore-amrnb-dev libopencore-amrwb-dev \
35+
libavresample-dev \
36+
x264 \
37+
libx264-dev \
38+
v4l-utils \
39+
libprotobuf-dev protobuf-compiler \
40+
libgoogle-glog-dev libgflags-dev \
41+
libgphoto2-dev \
42+
libeigen3-dev \
43+
libhdf5-dev \
44+
automake
45+
46+
47+
RUN update-alternatives --install /opt/conda/bin/python python /opt/conda/bin/python3 2
48+
49+
WORKDIR /tmp
50+
# install NLP packages *mecab-ko & khai*
51+
RUN curl -LO https://bitbucket.org/eunjeon/mecab-ko/downloads/mecab-0.996-ko-0.9.2.tar.gz && \
52+
tar zxfv mecab-0.996-ko-0.9.2.tar.gz && \
53+
cd mecab-0.996-ko-0.9.2 && \
54+
./configure && \
55+
make -j$(nproc) && \
56+
make check && \
57+
make install
58+
59+
RUN echo "Install mecab-ko-dic" && \
60+
cd /tmp && \
61+
ldconfig && \
62+
curl -LO https://bitbucket.org/eunjeon/mecab-ko-dic/downloads/mecab-ko-dic-2.1.1-20180720.tar.gz && \
63+
tar -zxvf mecab-ko-dic-2.1.1-20180720.tar.gz && \
64+
cd mecab-ko-dic-2.1.1-20180720 && \
65+
./autogen.sh && \
66+
./configure && \
67+
make -j$(nproc) && \
68+
sh -c 'echo "dicdir=/usr/local/lib/mecab/dic/mecab-ko-dic" > /usr/local/etc/mecabrc' && \
69+
make install && \
70+
cd /tmp && \
71+
git clone https://bitbucket.org/eunjeon/mecab-python-0.996.git && \
72+
python3 -m pip install /tmp/mecab-python-0.996
73+
74+
# OpenCV
75+
WORKDIR /tmp
76+
ENV OPENCV_VERSION="4.5.1"
77+
RUN ln -s /usr/include/libv4l1-videodev.h /usr/include/linux/videodev.h && \
78+
wget https://github.com/opencv/opencv/archive/${OPENCV_VERSION}.zip && \
79+
wget -O opencv-contrib.zip https://github.com/opencv/opencv_contrib/archive/${OPENCV_VERSION}.zip && \
80+
unzip ${OPENCV_VERSION}.zip && \
81+
unzip opencv-contrib.zip && \
82+
mkdir opencv-${OPENCV_VERSION}/cmake_binary && \
83+
cd opencv-${OPENCV_VERSION}/cmake_binary && \
84+
cmake \
85+
-DCMAKE_BUILD_TYPE=RELEASE \
86+
-D BUILD_TIFF=ON \
87+
-D BUILD_opencv_java=OFF \
88+
-D WITH_CUDA=ON \
89+
-D CUDA_NVCC_FLAGS=--expt-relaxed-constexpr \
90+
-D CUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda-11.1 \
91+
-D ENABLE_FAST_MATH=1 \
92+
-D CUDA_FAST_MATH=1 \
93+
-D WITH_CUBLAS=1 \
94+
-D WITH_OPENGL=ON \
95+
-D WITH_OPENCL=ON \
96+
-D WITH_IPP=ON \
97+
-D WITH_TBB=ON \
98+
-D WITH_EIGEN=ON \
99+
-D WITH_V4L=ON \
100+
-D BUILD_TESTS=OFF \
101+
-D BUILD_PERF_TESTS=OFF \
102+
-D OPENCV_EXTRA_MODULES_PATH="../../opencv_contrib-4.5.1/modules" \
103+
-D CMAKE_BUILD_TYPE=RELEASE \
104+
-D CMAKE_INSTALL_PREFIX=$(python3 -c "import sys; print(sys.prefix)") \
105+
-D PYTHON_EXECUTABLE=$(which python3) \
106+
-D PYTHON_INCLUDE_DIR=$(python3 -c "from distutils.sysconfig import get_python_inc; print(get_python_inc())") \
107+
-D PYTHON_PACKAGES_PATH=$(python3 -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())") \
108+
.. 2>&1 | tee cmake_messages.txt && \
109+
make -j$(nproc) && \
110+
make install && \
111+
cd /tmp && \
112+
python3 -m pip install --no-cache-dir opencv-python && \
113+
rm -fr opencv*
114+
WORKDIR /tmp
115+
RUN curl https://bootstrap.pypa.io/get-pip.py | python3 && \
116+
python3 -m pip install --no-cache-dir -U setuptools pip
117+
118+
RUN curl -sL https://deb.nodesource.com/setup_14.x | bash - && \
119+
apt-get update -y && \
120+
apt-get install -y nodejs
121+
122+
WORKDIR /tmp
123+
RUN git clone -q --branch=master git://github.com/xianyi/OpenBLAS.git && \
124+
cd OpenBLAS && \
125+
make DYNAMIC_ARCH=1 NO_AFFINITY=1 NUM_THREADS=48 FC=gfortran && \
126+
make install
127+
RUN git clone --recursive https://github.com/bodono/scs-python.git && \
128+
cd /tmp/scs-python && \
129+
python setup.py install --scs --gpu
130+
131+
RUN /opt/conda/bin/python3 -m pip install --no-cache-dir \
132+
Cython==0.29.21 \
133+
tornado==6.0.4 \
134+
pystan==2.19.1.1 \
135+
pycairo==1.19.0 \
136+
jupyter==1.0.0 \
137+
python-language-server[all] \
138+
matplotlib==3.3.3
139+
ENV SCIPY_VERSION 1.6.1
140+
# Install scipy
141+
RUN cd /tmp && \
142+
git clone --branch=v${SCIPY_VERSION} --depth=1 https://github.com/scipy/scipy.git scipy && \
143+
cd scipy && \
144+
git checkout -b v${SCIPY_VERSION} && \
145+
cp site.cfg.example site.cfg && \
146+
python3 -m pip install -U --no-cache-dir \
147+
numpy==1.20.1 \
148+
pandas==1.2.3 \
149+
scikit-learn==0.24.1 \
150+
hypothesis==6.4.2 \
151+
&& \
152+
python3 setup.py install
153+
154+
WORKDIR /tmp
155+
COPY ./requirements.txt /tmp
156+
RUN /opt/conda/bin/python3 -m pip install --no-cache-dir --ignore-installed -r requirements.txt && \
157+
/opt/conda/bin/python3 -m pip install --no-cache-dir tensorflow_model_analysis && \
158+
rm -f /tmp/*.whl /tmp/requirements.txt
159+
160+
# install git-lfs
161+
WORKDIR /tmp
162+
RUN curl -sLO https://github.com/git-lfs/git-lfs/releases/download/v2.13.2/git-lfs-linux-amd64-v2.13.2.tar.gz && \
163+
tar -zxf git-lfs-linux-amd64-v2.13.2.tar.gz && \
164+
bash install.sh && \
165+
rm -rf /tmp/*
166+
167+
COPY ./service-defs /etc/backend.ai/service-defs
168+
RUN curl -fL https://github.com/cdr/code-server/releases/download/v3.9.0/code-server-3.9.0-linux-amd64.tar.gz | tar -C /usr/local/lib -xz && \
169+
mv /usr/local/lib/code-server-3.9.0-linux-amd64 /usr/local/lib/code-server-3.9.0 && \
170+
ln -s /usr/local/lib/code-server-3.9.0/bin/code-server /usr/local/bin/code-server
171+
# Install Open MPI
172+
RUN mkdir /tmp/openmpi && \
173+
cd /tmp/openmpi && \
174+
wget https://www.open-mpi.org/software/ompi/v4.0/downloads/openmpi-4.0.5.tar.gz && \
175+
tar zxf openmpi-4.0.5.tar.gz && \
176+
cd openmpi-4.0.5 && \
177+
./configure --enable-orterun-prefix-by-default && \
178+
make -j $(nproc) all && \
179+
make install && \
180+
ldconfig && \
181+
rm -rf /tmp/openmpi*
182+
# Create a wrapper for OpenMPI to allow running as root by default
183+
RUN mv /usr/local/bin/mpirun /usr/local/bin/mpirun.real && \
184+
echo '#!/bin/bash' > /usr/local/bin/mpirun && \
185+
echo 'mpirun.real --allow-run-as-root "$@"' >> /usr/local/bin/mpirun && \
186+
chmod a+x /usr/local/bin/mpirun
187+
188+
# Configure OpenMPI to run good defaults:
189+
RUN echo "btl_tcp_if_exclude = lo,docker0" >> /usr/local/etc/openmpi-mca-params.conf
190+
191+
# Install Horovod, temporarily using CUDA stubs
192+
RUN ldconfig /usr/local/cuda/targets/x86_64-linux/lib/stubs && \
193+
HOROVOD_CUDA_HOME=$CONDA_PREFIX HOROVOD_GPU_ALLREDUCE=NCCL HOROVOD_GPU_BROADCAST=NCCL HOROVOD_NCCL_LINK=SHARED \
194+
HOROVOD_WITHOUT_TENSORFLOW=1 HOROVOD_WITH_PYTORCH=1 HOROVOD_WITHOUT_MXNET=1 \
195+
pip install --no-cache-dir horovod==0.21.3 && \
196+
ldconfig
197+
198+
RUN python3 -m pip install --no-cache-dir \
199+
mpi4py==3.0.3 \
200+
nni==2.0 \
201+
mlflow==1.14.1 \
202+
scikit-nni==0.2.1
203+
204+
RUN jupyter nbextensions_configurator enable && \
205+
jupyter contrib nbextension install && \
206+
jupyter nbextension enable --py --sys-prefix widgetsnbextension && \
207+
jupyter serverextension enable --py jupyterlab --sys-prefix && \
208+
jupyter labextension install --no-build @jupyter-widgets/jupyterlab-manager && \
209+
jupyter labextension install --no-build @krassowski/jupyterlab-lsp && \
210+
jupyter serverextension enable --py jupyter_lsp && \
211+
jupyter labextension install --no-build @jupyterlab/toc && \
212+
jupyter nbextension enable execute_time/ExecuteTime && \
213+
jupyter nbextension enable toc2/main && \
214+
jupyter labextension install @pyviz/jupyterlab_pyviz && \
215+
jupyter labextension install @bokeh/jupyter_bokeh && \
216+
jupyter labextension install --no-build jupyterlab-nvdashboard && \
217+
jupyter lab build
218+
219+
RUN apt autoclean && \
220+
sed -i 's/source \/usr\/local\/nvm\/nvm.sh//' /etc/bash.bashrc && \
221+
python3 -m pip uninstall -y tensorboard-plugin-wit && \
222+
rm -rf /var/lib/apt/lists/* && \
223+
rm -rf /root/.cache && \
224+
rm -rf /tmp/*
225+
226+
RUN /opt/conda/bin/python3 -m ipykernel install \
227+
--prefix=/opt/conda/ \
228+
--display-name "PyTorch 1.7 (NGC 21.02/Python 3.8 Conda) on Backend.AI" && \
229+
cat /opt/conda/share/jupyter/kernels/python3/kernel.json
230+
231+
# Backend.AI specifics
232+
LABEL ai.backend.kernelspec="1" \
233+
ai.backend.envs.corecount="OPENBLAS_NUM_THREADS,OMP_NUM_THREADS,NPROC" \
234+
ai.backend.features="batch query uid-match user-input" \
235+
ai.backend.base-distro="ubuntu16.04" \
236+
ai.backend.accelerators="cuda" \
237+
ai.backend.resource.min.cpu="1" \
238+
ai.backend.resource.min.mem="1g" \
239+
ai.backend.resource.min.cuda.device=1 \
240+
ai.backend.resource.min.cuda.shares=0.1 \
241+
ai.backend.base-distro="ubuntu16.04" \
242+
ai.backend.runtime-type="python" \
243+
ai.backend.runtime-path="/opt/conda/bin/python3" \
244+
ai.backend.service-ports="ipython:pty:3000,jupyter:http:8091,jupyterlab:http:8090,vscode:http:8180,tensorboard:http:6006,mlflow-ui:preopen:5000,nniboard:preopen:8080"
245+
246+
WORKDIR /home/work
247+
# vim: ft=dockerfile

‎vendor/ngc-pytorch/service-defs/tensorboard.json

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,8 @@
1010
"command": [
1111
"{runtime_path}",
1212
"-m", "tensorboard.main",
13-
"--host", "0.0.0.0",
1413
"--port", "{ports[0]}",
15-
"--debugger_port", "6064"
14+
"--bind_all"
1615
],
1716
"allowed_arguments": [
1817
"--logdir"

0 commit comments

Comments
 (0)