Skip to content

Commit 51cdf92

Browse files
hephaexhephaex
committed
fixed build error
2 parents 4c54d24 + d801b6b commit 51cdf92

19 files changed

+341
-156
lines changed

‎commons/Dockerfile.base.20.09-py36-cuda11

Lines changed: 39 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -31,56 +31,56 @@ RUN apt-get update -y && \
3131
apt-get update -y && \
3232
apt-get install -y --allow-downgrades --allow-change-held-packages --no-install-recommends \
3333
build-essential \
34-
ca-certificates \
34+
ca-certificates \
3535
gcc g++ make \
36-
gfortran \
37-
dkms \
36+
gfortran \
37+
dkms \
3838
git \
3939
curl \
4040
vim \
41-
yasm \
41+
yasm \
4242
wget zip unzip \
4343
software-properties-common \
44-
openssh-client openssh-server \
44+
openssh-client openssh-server \
4545
libssl-dev \
4646
libmpdec2 \
47-
pdsh curl net-tools \
48-
iputils-ping \
47+
pdsh curl net-tools \
48+
iputils-ping \
4949
libcurl3-dev \
5050
libfreetype6-dev \
5151
libhdf5-serial-dev \
5252
libzmq3-dev \
5353
libjpeg-dev \
5454
libpng-dev \
55-
libsm6 \
56-
libxext6 \
57-
libxrender-dev \
58-
libcairo2-dev libgirepository1.0-dev pkg-config gir1.2-gtk-3.0 \
55+
libsm6 \
56+
libxext6 \
57+
libxrender-dev \
58+
libcairo2-dev libgirepository1.0-dev pkg-config gir1.2-gtk-3.0 \
5959
libgeos-dev libgeos++-dev \
6060
pkg-config \
6161
zlib1g-dev \
62-
mime-support \
63-
proj-bin libproj-dev libgeos-dev libgeos++-dev graphviz \
62+
mime-support \
63+
proj-bin libproj-dev libgeos-dev libgeos++-dev graphviz \
6464
python${PYTHON_VERSION} \
6565
python${PYTHON_VERSION}-dev \
66-
libxml2-dev \
67-
libxslt1-dev \
68-
libasound2-dev \
69-
libnuma-dev \
70-
libjasper-dev \
66+
libxml2-dev \
67+
libxslt1-dev \
68+
libasound2-dev \
69+
libnuma-dev \
70+
libjasper-dev \
7171
libtiff-dev \
72-
libavcodec-dev \
72+
libavcodec-dev \
7373
libavformat-dev \
74-
libswscale-dev \
75-
libxine2-dev \
76-
libv4l-dev \
77-
libboost-dev \
78-
libboost-system-dev \
79-
libboost-filesystem-dev \
80-
xvfb \
81-
fonts-nanum \
82-
fonts-nanum-coding \
83-
fonts-nanum-extra \
74+
libswscale-dev \
75+
libxine2-dev \
76+
libv4l-dev \
77+
libboost-dev \
78+
libboost-system-dev \
79+
libboost-filesystem-dev \
80+
xvfb \
81+
fonts-nanum \
82+
fonts-nanum-coding \
83+
fonts-nanum-extra \
8484
&& \
8585
find /usr/local/cuda-${CUDA}/lib64/ -type f -name 'lib*_static.a' -not -name 'libcudart_static.a' -delete
8686

@@ -108,8 +108,7 @@ RUN curl -sL https://deb.nodesource.com/setup_10.x | bash - && \
108108

109109
RUN apt-get update && \
110110
apt-get install -y --no-install-recommends \
111-
libnvinfer${LIBNVINFER_MAJOR_VERSION}=${LIBNVINFER}+cuda${CUDA} \
112-
# libnvinfer-bin=${LIBNVINFER}+cuda${CUDA} \
111+
libnvinfer${LIBNVINFER_MAJOR_VERSION}=${LIBNVINFER}+cuda${CUDA} \
113112
libnvinfer-dev=${LIBNVINFER}+cuda${CUDA} \
114113
libnvinfer-plugin-dev=${LIBNVINFER}+cuda${CUDA} \
115114
&& \
@@ -167,7 +166,7 @@ COPY ./numpy-1.18.5-cp36-cp36m-manylinux2010_x86_64.whl /tmp
167166
COPY ./pandas-1.0.4+4.g29edbab4a-cp36-cp36m-linux_x86_64.whl /tmp
168167

169168
RUN python3 -m pip install --no-cache-dir --upgrade -Iv \
170-
Cython==0.29.20 \
169+
Cython==0.29.20 \
171170
/tmp/numpy-1.18.5-cp36-cp36m-manylinux2010_x86_64.whl \
172171
/tmp/pandas-1.0.4+4.g29edbab4a-cp36-cp36m-linux_x86_64.whl \
173172
tornado==6.0.4 \
@@ -186,25 +185,26 @@ RUN git clone --recursive https://github.com/bodono/scs-python.git && \
186185
python setup.py install --scs --gpu
187186

188187
RUN python3 -m pip install pip --no-cache-dir \
189-
Cartopy==0.18.0 \
188+
Cartopy==0.18.0 \
190189
notebook==6.0.3
191190

192191
WORKDIR /tmp
193192
COPY ./requirements.20.08.txt /tmp
194193
RUN python3 -m pip install --no-cache-dir --upgrade -r requirements.20.08.txt && \
195194
python3 -m pip install --no-cache-dir tensorflow_model_analysis && \
196-
python3 -m pip uninstall -y tensorboard tensorboard-plugin-wit tensorflow tensorflow-gpu \
195+
python3 -m pip uninstall -y \
196+
tensorboard tensorboard-plugin-wit tensorflow tensorflow-gpu \
197197
tensorflow-estimator tensorflow-addons tensorflow-datasets \
198-
tensorflow-gan tensorflow-hub tensorflow-metadata \
199-
tensorflow-probability tensorflow-text \
200-
tensorflow-model-analysis tensorflow-serving-api && \
198+
tensorflow-gan tensorflow-hub tensorflow-metadata \
199+
tensorflow-probability tensorflow-text \
200+
tensorflow-model-analysis tensorflow-serving-api && \
201201
rm -f /tmp/*.whl /tmp/requirements.20.08.txt
202202

203203
# install NLP packages *mecab-ko & khai*
204204
RUN apt-get update && \
205205
apt-get install -y \
206206
openjdk-8-jdk \
207-
automake \
207+
automake \
208208
&& \
209209
cd /tmp && \
210210
curl -LO https://bitbucket.org/eunjeon/mecab-ko/downloads/mecab-0.996-ko-0.9.2.tar.gz && \
@@ -236,7 +236,7 @@ RUN ln -s /usr/include/libv4l1-videodev.h /usr/include/linux/videodev.h && \
236236
libgstreamer1.0-dev \
237237
libgstreamer-plugins-base1.0-dev \
238238
libgtk-3-dev \
239-
libtbb-dev \
239+
libtbb-dev \
240240
libatlas-base-dev \
241241
libdc1394-22-dev \
242242
libxvidcore-dev \
@@ -380,13 +380,9 @@ RUN jupyter nbextensions_configurator enable && \
380380
jupyter nbextension enable --py --sys-prefix widgetsnbextension && \
381381
jupyter serverextension enable --py jupyterlab --sys-prefix && \
382382
jupyter labextension install --no-build @jupyter-widgets/jupyterlab-manager && \
383-
# git clone https://github.com/lambdalisue/jupyter-vim-binding vim_binding && \
384-
# jupyter nbextension enable /home/work/.jupyter/nbextension/vim_binding/vim_binding && \
385-
# jupyter labextension install --no-build @lckr/jupyterlab_variableinspector && \
386383
jupyter labextension install --no-build @krassowski/jupyterlab-lsp && \
387384
jupyter serverextension enable --py jupyter_lsp && \
388385
jupyter labextension install --no-build @jupyterlab/toc && \
389-
# jupyter labextension install @jupyterlab/hdf5
390386
jupyter nbextension enable execute_time/ExecuteTime && \
391387
jupyter nbextension enable toc2/main && \
392388
jupyter labextension install @pyviz/jupyterlab_pyviz && \

‎commons/Dockerfile.base.py38-cuda11.1

Lines changed: 9 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,7 @@ RUN git clone https://github.com/Mellanox/nv_peer_memory.git /tmp/nv_peer_memory
161161
dpkg-buildpackage -us -uc && \
162162
dpkg -i /tmp/nvidia-peer-memory_1.1-0_all.deb
163163

164+
# Oracle client
164165
RUN mkdir -p /opt/oracle && \
165166
cd /opt/oracle && \
166167
apt-get clean && \
@@ -186,19 +187,15 @@ RUN rm /usr/bin/python /usr/bin/python3 && \
186187

187188
WORKDIR /tmp
188189
RUN curl https://bootstrap.pypa.io/get-pip.py | python3 && \
189-
python3 -m pip install --no-cache-dir -U setuptools pip==20.3.2
190-
191-
RUN python3 -m pip install --no-cache-dir \
192-
Cython==0.29.21 \
193-
tornado==6.1 \
194-
nvidia-pyindex==1.0.5 \
195-
matplotlib==3.3.3
190+
python3 -m pip install --no-cache-dir -U setuptools pip
196191

197192
WORKDIR /tmp
193+
# OpenBlas
198194
RUN git clone -q --branch=master git://github.com/xianyi/OpenBLAS.git && \
199195
cd OpenBLAS && \
200196
make DYNAMIC_ARCH=1 NO_AFFINITY=1 NUM_THREADS=48 FC=gfortran && \
201197
make install
198+
# Numpy
202199
RUN git clone --branch=v${NUMPY_VERSION} --depth=1 https://github.com/numpy/numpy.git numpy && \
203200
cd numpy && \
204201
git checkout -b v${NUMPY_VERSION} && \
@@ -210,7 +207,7 @@ RUN git clone --branch=v${NUMPY_VERSION} --depth=1 https://github.com/numpy/nump
210207
echo "lapack_libs =" >> site.cfg && \
211208
python3 setup.py bdist_wheel -d /tmp
212209

213-
# Install scipy
210+
# Scipy
214211
RUN cd /tmp && \
215212
git clone --branch=v${SCIPY_VERSION} --depth=1 https://github.com/scipy/scipy.git scipy && \
216213
cd scipy && \
@@ -257,11 +254,6 @@ RUN echo "Install mecab-ko-dic" && \
257254
git clone https://bitbucket.org/eunjeon/mecab-python-0.996.git && \
258255
python3 -m pip install /tmp/mecab-python-0.996
259256

260-
RUN python3 -m pip install pip --no-cache-dir \
261-
Cartopy==0.18.0 \
262-
nvidia-dlprof \
263-
notebook==6.0.3
264-
265257
WORKDIR /tmp
266258
COPY ./requirements.py38.txt /tmp
267259
RUN python3 -m pip install --no-cache-dir -r requirements.py38.txt && \
@@ -408,18 +400,20 @@ RUN cat /etc/ssh/ssh_config | grep -v StrictHostKeyChecking > /etc/ssh/ssh_confi
408400
echo " StrictHostKeyChecking no" >> /etc/ssh/ssh_config.new && \
409401
mv /etc/ssh/ssh_config.new /etc/ssh/ssh_config
410402

411-
# install git-lfs
403+
# Install git-lfs
412404
WORKDIR /tmp
413405
RUN curl -sLO https://github.com/git-lfs/git-lfs/releases/download/v2.13.1/git-lfs-linux-amd64-v2.13.1.tar.gz && \
414406
tar -zxf git-lfs-linux-amd64-v2.13.1.tar.gz && \
415407
bash install.sh && \
416408
rm -rf /tmp/*
417409

410+
# Install VSCode
418411
RUN curl -fL https://github.com/cdr/code-server/releases/download/v3.8.0/code-server-3.8.0-linux-amd64.tar.gz \
419412
| tar -C /usr/local/lib -xz && \
420413
mv /usr/local/lib/code-server-3.8.0-linux-amd64 /usr/local/lib/code-server-3.8.0 && \
421414
ln -s /usr/local/lib/code-server-3.8.0/bin/code-server /usr/local/bin/code-server
422415

416+
# Install Jupyterlab extensions
423417
RUN jupyter nbextensions_configurator enable && \
424418
jupyter contrib nbextension install && \
425419
jupyter nbextension enable --py --sys-prefix widgetsnbextension && \
@@ -428,18 +422,17 @@ RUN jupyter nbextensions_configurator enable && \
428422
jupyter labextension install --no-build @krassowski/jupyterlab-lsp && \
429423
jupyter serverextension enable --py jupyter_lsp && \
430424
jupyter labextension install --no-build @jupyterlab/toc && \
431-
# jupyter labextension install @jupyterlab/hdf5
432425
jupyter nbextension enable execute_time/ExecuteTime && \
433426
jupyter nbextension enable toc2/main && \
434427
jupyter labextension install @pyviz/jupyterlab_pyviz && \
435428
jupyter labextension install @bokeh/jupyter_bokeh && \
436429
jupyter labextension install --no-build jupyterlab-nvdashboard && \
437430
jupyter lab build
438431

432+
# Clean up
439433
RUN apt autoclean && \
440434
ln -s /usr/local/cuda-11.1/targets/x86_64-linux/lib/libcusolver.so.11 /usr/local/cuda-11.1/targets/x86_64-linux/lib/libcusolver.so.10 && \
441435
sed -i 's/source \/usr\/local\/nvm\/nvm.sh//' /etc/bash.bashrc && \
442-
python3 -m pip uninstall -y tensorboard-plugin-wit && \
443436
rm -rf /var/lib/apt/lists/* && \
444437
rm -rf /root/.cache && \
445438
rm -rf /tmp/*

‎commons/requirements.py38.txt

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ boto3==1.9.209
2525
botocore==1.12.209
2626
branca==0.4.0
2727
cachetools==3.1.1
28+
Cartopy==0.18.0
2829
catboost==0.22
2930
category-encoders==2.0.0
3031
certifi==2019.11.28
@@ -43,9 +44,8 @@ coverage==4.5.4
4344
cryptography==3.0
4445
cvxpy==1.0.24
4546
cycler==0.10.0
46-
Cython==0.29.13
47+
Cython==0.29.21
4748
dask==2.9.0
48-
#dataclasses==0.8
4949
deap==1.3.1
5050
decorator==4.4.2
5151
defusedxml==0.6.0
@@ -140,6 +140,7 @@ lxml==4.5.0
140140
Mako==1.0.7
141141
Markdown==3.2.1
142142
MarkupSafe==1.1.1
143+
matplotlib==3.3.3
143144
mccabe==0.6.1
144145
mistune==0.8.4
145146
mlxtend==0.17.2
@@ -160,12 +161,14 @@ networkx==2.4
160161
#ngraph-onnx==0.24.0
161162
#ngraph-tensorflow-bridge==0.18.0
162163
nltk==3.2.5
164+
notebook==6.0.3
163165
numba==0.48.0
164166
numexpr==2.7.0
167+
nvidia-dlprof==0.0.1.dev4
168+
nvidia-pyindex==1.0.5
165169
oauthlib==3.1.0
166170
oauth2client==3.0.0
167171
onnx==1.8.0
168-
#onnx-plaidml==0.6.3
169172
opt-einsum==3.3.0
170173
osqp==0.6.2
171174
packaging==20.3
@@ -179,8 +182,7 @@ pexpect==4.8.0
179182
pickleshare==0.7.5
180183
Pillow==6.2.2
181184
plaidml==0.7.0
182-
#plaidml-keras==0.7.0
183-
plotly==4.5.3
185+
Splotly==4.5.3
184186
pluggy==0.12.0
185187
ply==3.11
186188
portpicker==1.3.1
@@ -254,7 +256,6 @@ scikit-plot==0.3.7
254256
scikit-optimize==0.7.4
255257
scikit-surprise==1.1.0
256258
scooby==0.5.5
257-
#scs==2.1.0
258259
seaborn==0.10.0
259260
selenium==3.141.0
260261
Send2Trash==1.5.0
@@ -285,6 +286,7 @@ tf2onnx==1.5.5
285286
toml==0.10.0
286287
toolz==0.10.0
287288
torch==1.7.1
289+
tornado==6.1
288290
TPOT==0.11.1
289291
tqdm==4.55.1
290292
traitlets==4.3.3

‎python-pytorch/Dockerfile.1.7-py38-cuda11.1

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
FROM lablup/common-base:py38-cuda11.1
2+
# Install PyTorch and MXNet
3+
ENV PYTORCH_VERSION=1.7.1
4+
ENV TORCHVISION_VERSION=0.8.2
5+
ENV TORCHAUDIO_VERSION=0.7.2
6+
ENV TORCHTEXT_VERSION=0.8.1
7+
ENV TENSORBOARDX_VERSION=2.1
8+
ENV MXNET_VERSION=1.6.0
9+
10+
RUN python3 -m pip uninstall -y torch && \
11+
python3 -m pip install --no-cache-dir \
12+
https://download.pytorch.org/whl/cu110/torch-${PYTORCH_VERSION}%2Bcu110-cp38-cp38-linux_x86_64.whl \
13+
https://download.pytorch.org/whl/cu110/torchvision-${TORCHVISION_VERSION}%2Bcu110-cp38-cp38-linux_x86_64.whl \
14+
https://download.pytorch.org/whl/torchaudio-${TORCHAUDIO_VERSION}-cp38-cp38-linux_x86_64.whl \
15+
https://download.pytorch.org/whl/torchtext-${TORCHTEXT_VERSION}-cp38-cp38-linux_x86_64.whl && \
16+
python3 -m pip install --no-cache-dir tensorboardX==${TENSORBOARDX_VERSION}
17+
18+
# torch2trt PyTorch to TensorRT converter which utilizes the TensorRT Python API
19+
WORKDIR /tmp
20+
RUN git clone https://github.com/NVIDIA-AI-IOT/torch2trt /tmp/torch2trt && \
21+
cd /tmp/torch2trt && \
22+
python3 setup.py install --plugins && \
23+
rm -fr /tmp/torch2trt
24+
25+
RUN python3 -m pip install --extra-index-url \
26+
https://developer.download.nvidia.com/compute/redist nvidia-dali-cuda100
27+
28+
# Install Horovod, temporarily using CUDA stubs
29+
RUN ldconfig /usr/local/cuda/targets/x86_64-linux/lib/stubs && \
30+
HOROVOD_GPU_ALLREDUCE=NCCL HOROVOD_GPU_BROADCAST=NCCL \
31+
HOROVOD_WITH_TENSORFLOW=1 HOROVOD_WITH_PYTORCH=1 HOROVOD_WITHOUT_MXNET=1 \
32+
pip install --no-cache-dir horovod==0.21.1 && \
33+
ldconfig
34+
35+
RUN python3 -m pip install --no-cache-dir \
36+
mpi4py==3.0.3 \
37+
mlflow==1.12.1 \
38+
nni==1.9 \
39+
scikit-nni==0.2.1
40+
41+
RUN apt autoclean && \
42+
rm -rf /var/lib/apt/lists/* && \
43+
rm -rf /root/.cache && \
44+
rm -rf /tmp/*
45+
COPY ./service-defs /etc/backend.ai/service-defs
46+
47+
# Install ipython kernelspec
48+
Run python3 -m ipykernel install --display-name "PyTorch 1.7.1 on Python 3.8 (CUDA 11.1)" && \
49+
cat /usr/local/share/jupyter/kernels/python3/kernel.json
50+
51+
# Backend.AI specifics
52+
LABEL ai.backend.kernelspec="1" \
53+
ai.backend.envs.corecount="OPENBLAS_NUM_THREADS,OMP_NUM_THREADS,NPROC" \
54+
ai.backend.features="batch query uid-match user-input" \
55+
ai.backend.base-distro="ubuntu16.04" \
56+
ai.backend.resource.min.cpu="1" \
57+
ai.backend.resource.min.mem="1g" \
58+
ai.backend.resource.min.cuda.device=0 \
59+
ai.backend.resource.min.cuda.shares=0 \
60+
ai.backend.runtime-type="python" \
61+
ai.backend.runtime-path="/usr/bin/python3" \
62+
ai.backend.service-ports="ipython:pty:3000,jupyter:http:8081,jupyterlab:http:8090,vscode:http:8180,tensorboard:http:6006,mlflow-ui:preopen:5000,nniboard:preopen:8080"
63+
64+
WORKDIR /home/work
65+
# vim: ft=dockerfile

‎python-pytorch/service-defs/digits.json

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"command": [
3+
"{runtime_path}", "-m", "digits"
4+
]
5+
}

0 commit comments

Comments
 (0)