Skip to content

Commit ba94cf1

Browse files
hephaexhephaex
committed
Merge branch '20.12' of https://github.com/lablup/backend.ai-kernels into 20.12
2 parents fac7240 + 86b7834 commit ba94cf1

File tree

6 files changed

+62
-61
lines changed

6 files changed

+62
-61
lines changed

‎README.md

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ Here we list the latest versions of our supported kernel images.
1313
| Deep-Learning Framework | Image Name | Version | Batch | Query | Input Hook | TTY | Runtime Impl. |
1414
|-------------------------|----------------------------|---------|-------|-------|------------|-----|-------------------|
1515
| TensorFlow | `lablup/python-tensorflow` | 1.14.0 | O | O\* | O | | Bundled w/Keras 2 |
16-
| TensorFlow | `lablup/python-tensorflow` | 1.15.0 | O | O\* | O | | Bundled w/Keras 2 |
16+
| TensorFlow | `lablup/python-tensorflow` | 1.15.4 | O | O\* | O | | Bundled w/Keras 2 |
1717
| TensorFlow | `lablup/python-tensorflow` | 2.0.3 | O | O\* | O | | Bundled w/Keras 2 |
1818
| TensorFlow | `lablup/python-tensorflow` | 2.1.2 | O | O\* | O | | Bundled w/Keras 2 |
1919
| TensorFlow | `lablup/python-tensorflow` | 2.2.1 | O | O\* | O | | Bundled w/Keras 2 |
@@ -22,12 +22,13 @@ Here we list the latest versions of our supported kernel images.
2222
| PyTorch | `lablup/python-torch` | 1.4 | O | O\* | O | | |
2323
| PyTorch | `lablup/python-torch` | 1.5 | O | O\* | O | | |
2424
| PyTorch | `lablup/python-torch` | 1.6 | O | O\* | O | | |
25+
| PyTorch | `lablup/python-torch` | 1.7 | O | O\* | O | | |
2526
| CNTK | `lablup/python-cntk` | 2.6 | O | O\* | O | | |
2627
| CNTK | `lablup/python-cntk` | 2.7 | O | O\* | O | | |
2728
| MXnet | `lablup/python-mxnet` | 1.4.1 | O | O\* | O | | Bundled w/Keras 2 |
2829
| MXnet | `lablup/python-mxnet` | 1.5.1 | O | O\* | O | | Bundled w/Keras 2 |
2930
| MXnet | `lablup/python-mxnet` | 1.6.0 | O | O\* | O | | Bundled w/Keras 2 |
30-
| All-in-one Environment | `lablup/python-ff` | 20.07 | O | O\* | O | | |
31+
| All-in-one Environment | `lablup/python-ff` | 20.09 | O | O\* | O | | |
3132

3233
### Warrented Images (NGC, Nvidia GPU Cloud)
3334
| Deep-Learning Framework | Image Name | Version | Batch | Query | Input Hook | TTY | Runtime Impl. |
@@ -58,6 +59,10 @@ Here we list the latest versions of our supported kernel images.
5859
| NGC-TensorFlow | `lablup/ngc-tensorflow` | 20.08-tf2-py3 | O | O\* | O | | Bundled w/Keras 2 |
5960
| NGC-TensorFlow | `lablup/ngc-tensorflow` | 20.09-tf1-py3 | O | O\* | O | | Bundled w/Keras 2 |
6061
| NGC-TensorFlow | `lablup/ngc-tensorflow` | 20.09-tf2-py3 | O | O\* | O | | Bundled w/Keras 2 |
62+
| NGC-TensorFlow | `lablup/ngc-tensorflow` | 20.10-tf1-py3 | O | O\* | O | | Bundled w/Keras 2 |
63+
| NGC-TensorFlow | `lablup/ngc-tensorflow` | 20.10-tf2-py3 | O | O\* | O | | Bundled w/Keras 2 |
64+
| NGC-TensorFlow | `lablup/ngc-tensorflow` | 20.11-tf1-py3 | O | O\* | O | | Bundled w/Keras 2 |
65+
| NGC-TensorFlow | `lablup/ngc-tensorflow` | 20.11-tf2-py3 | O | O\* | O | | Bundled w/Keras 2 |
6166
| NGC-Pytorch | `lablup/ngc-pytorch` | 19.07-py3 | O | O\* | O | | |
6267
| NGC-Pytorch | `lablup/ngc-pytorch` | 19.08-py3 | O | O\* | O | | |
6368
| NGC-Pytorch | `lablup/ngc-pytorch` | 19.09-py3 | O | O\* | O | | |
@@ -71,6 +76,8 @@ Here we list the latest versions of our supported kernel images.
7176
| NGC-Pytorch | `lablup/ngc-pytorch` | 20.07-py3 | O | O\* | O | | |
7277
| NGC-Pytorch | `lablup/ngc-pytorch` | 20.08-py3 | O | O\* | O | | |
7378
| NGC-Pytorch | `lablup/ngc-pytorch` | 20.09-py3 | O | O\* | O | | |
79+
| NGC-Pytorch | `lablup/ngc-pytorch` | 20.10-py3 | O | O\* | O | | |
80+
| NGC-Pytorch | `lablup/ngc-pytorch` | 20.11-py3 | O | O\* | O | | |
7481
| NGC-MXNet | `lablup/ngc-mxnet` | 19.07-py3 | O | O\* | O | | Bundled w/Keras 2 |
7582
| NGC-MXNet | `lablup/ngc-mxnet` | 19.08-py3 | O | O\* | O | | Bundled w/Keras 2 |
7683
| NGC-MXNet | `lablup/ngc-mxnet` | 19.09-py3 | O | O\* | O | | Bundled w/Keras 2 |

‎commons/Dockerfile.base.cuda11.1

Lines changed: 21 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ ARG CUDNN=8.0.5.39-1
88
ENV NCCL=2.8.3
99
ENV NCCL_VERSION=2.8.3-1+cuda11.1
1010
ENV CUDNN_VERSION 8.0.5.43
11-
ENV MLNX_OFED_VERSION=5.1-2.3.7
11+
ENV MLNX_OFED_VERSION=5.1-2.3.7.1
1212
ENV MKL_VERSION=2020.3
1313
ENV MKL_BUILD=279
1414
ENV NUMPY_VERSION 1.19.4
@@ -145,25 +145,23 @@ RUN mkdir -p /etc/OpenCL/vendors && \
145145
echo "libnvidia-opencl.so.1" > /etc/OpenCL/vendors/nvidia.icd
146146

147147
# OFED
148-
ENV STAGE_DIR=/tmp
149-
RUN mkdir -p ${STAGE_DIR}
150-
RUN cd ${STAGE_DIR} && \
151-
wget -q -O - http://www.mellanox.com/downloads/ofed/MLNX_OFED-${MLNX_OFED_VERSION}/MLNX_OFED_LINUX-${MLNX_OFED_VERSION}-ubuntu18.04-x86_64.tgz | tar xzf -
152-
RUN cd ${STAGE_DIR}/MLNX_OFED_LINUX-${MLNX_OFED_VERSION}-ubuntu18.04-x86_64 && \
148+
WORKDIR /tmp
149+
RUN wget -q -O - https://www.mellanox.com/downloads/ofed/MLNX_OFED-5.1-2.3.7.1/MLNX_OFED_LINUX-${MLNX_OFED_VERSION}-ubuntu18.04-x86_64.tgz | tar xzf - && \
150+
cd /tmp/MLNX_OFED_LINUX-${MLNX_OFED_VERSION}-ubuntu18.04-x86_64 && \
153151
./mlnxofedinstall --user-space-only --without-fw-update --all -q && \
154-
cd ${STAGE_DIR} && \
155-
rm -rf ${STAGE_DIR}/MLNX_OFED_LINUX-${MLNX_OFED_VERSION}-ubuntu18.04-x86_64*
152+
cd /tmp && \
153+
rm -rf /tmp/MLNX_OFED_LINUX-${MLNX_OFED_VERSION}-ubuntu18.04-x86_64*
156154

157155
# nvidia-peer
158-
RUN mkdir -p ${STAGE_DIR} && \
159-
git clone https://github.com/Mellanox/nv_peer_memory.git ${STAGE_DIR}/nv_peer_memory && \
160-
cd ${STAGE_DIR}/nv_peer_memory && \
156+
WORKDIR /tmp
157+
RUN git clone https://github.com/Mellanox/nv_peer_memory.git /tmp/nv_peer_memory && \
158+
cd /tmp/nv_peer_memory && \
161159
./build_module.sh && \
162-
cd ${STAGE_DIR} && \
163-
tar xzf ${STAGE_DIR}/nvidia-peer-memory_1.1.orig.tar.gz && \
164-
cd ${STAGE_DIR}/nvidia-peer-memory-1.1 && \
160+
cd /tmp && \
161+
tar xzf /tmp/nvidia-peer-memory_1.1.orig.tar.gz && \
162+
cd /tmp/nvidia-peer-memory-1.1 && \
165163
dpkg-buildpackage -us -uc && \
166-
dpkg -i ${STAGE_DIR}/nvidia-peer-memory_1.1-0_all.deb
164+
dpkg -i /tmp/nvidia-peer-memory_1.1-0_all.deb
167165

168166
RUN mkdir -p /opt/oracle && \
169167
cd /opt/oracle && \
@@ -203,10 +201,6 @@ RUN git clone -q --branch=master git://github.com/xianyi/OpenBLAS.git && \
203201
cd OpenBLAS && \
204202
make DYNAMIC_ARCH=1 NO_AFFINITY=1 NUM_THREADS=48 FC=gfortran && \
205203
make install
206-
RUN cd /tmp && \
207-
git clone --recursive https://github.com/bodono/scs-python.git && \
208-
cd scs-python && \
209-
python setup.py install --scs --gpu
210204
RUN git clone --branch=v${NUMPY_VERSION} --depth=1 https://github.com/numpy/numpy.git numpy && \
211205
cd numpy && \
212206
git checkout -b v${NUMPY_VERSION} && \
@@ -229,8 +223,12 @@ RUN cd /tmp && \
229223
echo "library_dirs = /opt/intel/compilers_and_libraries_2020.3.279/linux/mkl/lib/intel64" >> site.cfg && \
230224
echo "mkl_libs = mkl_rt" >> site.cfg && \
231225
echo "lapack_libs =" >> site.cfg && \
232-
python3 -m pip install -U --no-cache-dir /tmp/numpy-1.18.5-cp37-cp37m-linux_x86_64.whl && \
226+
python3 -m pip install -U --no-cache-dir /tmp/numpy-1.19.4-cp36-cp36m-linux_x86_64.whl && \
233227
python3 setup.py install
228+
RUN cd /tmp && \
229+
git clone --recursive https://github.com/bodono/scs-python.git && \
230+
cd scs-python && \
231+
python setup.py install --scs --gpu
234232

235233
# install NLP packages *mecab-ko & khai*
236234
RUN apt-get update && \
@@ -264,6 +262,7 @@ RUN echo "Install mecab-ko-dic" && \
264262

265263
RUN python3 -m pip install pip --no-cache-dir \
266264
Cartopy==0.18.0 \
265+
nvidia-dlprof \
267266
notebook==6.0.3
268267

269268
WORKDIR /tmp
@@ -309,7 +308,7 @@ RUN wget https://github.com/Kitware/CMake/releases/download/v3.19.2/cmake-3.19.2
309308
rm /tmp/cmake-install.sh
310309

311310
WORKDIR /tmp
312-
ENV OPENCV_VERSION="4.5.0"
311+
ENV OPENCV_VERSION="4.5.1"
313312
RUN wget https://github.com/opencv/opencv/archive/${OPENCV_VERSION}.zip && \
314313
wget -O opencv-contrib.zip https://github.com/opencv/opencv_contrib/archive/${OPENCV_VERSION}.zip && \
315314
unzip ${OPENCV_VERSION}.zip && \
@@ -334,7 +333,7 @@ RUN wget https://github.com/opencv/opencv/archive/${OPENCV_VERSION}.zip && \
334333
-D WITH_V4L=ON \
335334
-D BUILD_TESTS=OFF \
336335
-D BUILD_PERF_TESTS=OFF \
337-
-D OPENCV_EXTRA_MODULES_PATH="../../opencv_contrib-4.5.0/modules" \
336+
-D OPENCV_EXTRA_MODULES_PATH="../../opencv_contrib-4.5.1/modules" \
338337
-D CMAKE_BUILD_TYPE=RELEASE \
339338
-D CMAKE_INSTALL_PREFIX=$(python3 -c "import sys; print(sys.prefix)") \
340339
-D PYTHON_EXECUTABLE=$(which python3) \

‎commons/requirements.txt

Lines changed: 30 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
1-
absl-py==0.8.0
1+
absl-py==0.10.0
22
aiohttp==3.6.2
33
aiozmq==0.9.0
4-
adanet==0.8.0
4+
adanet==0.5.0
5+
apache-beam==2.24.0
56
appdirs==1.4.3
67
astor==0.8.1
7-
astroid==2.3.3
8+
astroid==2.3.0
89
async-timeout==3.0.1
910
atlas==0.27.0
1011
atomicwrites==1.3.0
@@ -16,14 +17,14 @@ avro-python3==1.9.1
1617
backcall==0.1.0
1718
bayesian-optimization==1.1.0
1819
beautifulsoup4==4.6.0
19-
black==19.10b0
20+
black==19.3b0
2021
bleach==3.1.1
2122
bokeh==1.4.0
2223
boto==2.49.0
2324
boto3==1.9.209
2425
botocore==1.12.209
2526
branca==0.4.0
26-
cachetools==4.0.0
27+
cachetools==3.1.1
2728
catboost==0.22
2829
category-encoders==2.0.0
2930
certifi==2019.11.28
@@ -65,16 +66,16 @@ flake8==3.7.9
6566
folium==0.10.0
6667
funcy==1.13
6768
future==0.18.2
68-
gast==0.2.2
69+
gast==0.3.3
6970
gensim==3.8.0
7071
geopandas==0.5.1
7172
geoplotlib==0.3.2
7273
google-auth==1.18.0
7374
google-auth-oauthlib==0.4.1
7475
google-images-download==2.8.0
75-
google-pasta==0.1.8
76+
google-pasta==0.2
7677
graphviz==0.14
77-
grpcio==1.29.0
78+
grpcio==1.32.0
7879
gym==0.17.3
7980
h5py==2.10.0
8081
hdf5storage==0.1.15
@@ -120,7 +121,7 @@ jupyterlab_hdf==0.3.0
120121
jupyterthemes==0.20.0
121122
Keras==2.3.1
122123
Keras-Applications==1.0.8
123-
Keras-Preprocessing==1.1.0
124+
Keras-Preprocessing==1.1.2
124125
kss==1.3.1
125126
kiwisolver==1.1.0
126127
konlpy==0.5.2
@@ -141,7 +142,7 @@ mlxtend==0.17.2
141142
mmdnn==0.3.1
142143
mne==0.20.5
143144
more-itertools==7.2.0
144-
mock==3.0.5
145+
mock==2.0.0
145146
msgpack==0.6.1
146147
msgpack-python==0.5.6
147148
multidict==4.7.5
@@ -157,18 +158,18 @@ ngraph-tensorflow-bridge==0.18.0
157158
nltk==3.2.5
158159
numba==0.48.0
159160
numexpr==2.7.0
160-
nvidia-dlprof==0.0.1.dev4
161161
oauthlib==3.1.0
162162
oauth2client==3.0.0
163163
onnx==1.6.0
164164
#onnx-plaidml==0.6.3
165-
opt-einsum==3.2.0
165+
opt-einsum==3.3.0
166166
osqp==0.5.0
167167
packaging==20.3
168168
pandocfilters==1.4.2
169169
parso==0.7.0
170170
pathspec==0.7.0
171171
patsy==0.5.1
172+
pbr==5.5.0
172173
pexpect==4.8.0
173174
pickleshare==0.7.5
174175
Pillow==6.2.2
@@ -180,15 +181,15 @@ ply==3.11
180181
portpicker==1.3.1
181182
progressbar==2.5
182183
prometheus-client==0.7.1
183-
prompt-toolkit==3.0.3
184+
prompt-toolkit==2.0.10
184185
protobuf==3.12.2
185186
psutil==5.7.0
186187
ptyprocess==0.6.0
187188
py==1.8.0
188-
#pyaml==19.12.0
189+
pyaml==19.12.0
189190
pyasn1==0.4.8
190191
pyasn1-modules==0.2.8
191-
pyarrow==0.15.1
192+
pyarrow==0.17.0
192193
pycocotools==2.0.0
193194
pycodestyle==2.5.0
194195
pycparser==2.19
@@ -211,6 +212,7 @@ pyqt5==5.15.0
211212
pyrsistent==0.15.7
212213
pyshp==2.1.0
213214
PySocks==1.7.1
215+
pystan==2.19.1.1
214216
pytest==5.4.3
215217
python-dateutil==2.8.1
216218
python-jsonrpc-server==0.3.4
@@ -226,7 +228,7 @@ pyzmq==19.0.0
226228
qtconsole==4.7.1
227229
QtPy==1.9.0
228230
regex==2020.2.20
229-
requests==2.23.0
231+
requests==2.24.0
230232
requests-oauthlib==1.3.0
231233
resampy==0.2.1
232234
retrying==1.3.3
@@ -237,12 +239,15 @@ ruamel.yaml.clib==0.2.0
237239
s3transfer==0.2.1
238240
sacred==0.8.1
239241
sacrebleu==1.3.6
242+
scipy==1.5.4
240243
schema==0.7.1
241244
scikit-image==0.16.2
242245
scikit-learn==0.23.1
243246
scikit-plot==0.3.7
244247
scikit-optimize==0.7.4
245248
scikit-surprise==1.1.0
249+
scooby==0.5.5
250+
scs==2.1.0
246251
seaborn==0.10.0
247252
selenium==3.141.0
248253
Send2Trash==1.5.0
@@ -252,7 +257,7 @@ Shapely==1.7.0
252257
simpervisor==0.3
253258
simpleaudio==1.0.4
254259
simplejson==3.17.0
255-
six==1.14.0
260+
six==1.15.0
256261
smart-open==1.9.0
257262
snowballstemmer==2.0.0
258263
sortedcontainers==2.1.0
@@ -261,20 +266,24 @@ statsmodels==0.11.1
261266
stopit==1.1.2
262267
tabulate==0.8.7
263268
tblib==1.6.0
264-
tensorflow_model_analysis==0.26.0
269+
tensorboardx==2.1
270+
tensorflow==2.4.0
271+
tensorflowjs==2.8.0
272+
tensorflow_model_analysis==0.24.0
265273
termcolor==1.1.0
266274
terminado==0.8.3
267275
test-generator==0.1.1
268276
testpath==0.4.4
269277
tf2onnx==1.5.5
270278
toml==0.10.0
271279
toolz==0.10.0
280+
torch==1.7.1
272281
TPOT==0.11.1
273282
tqdm==4.43.0
274283
traitlets==4.3.3
275284
trax==1.3.6
276285
tweepy==3.8.0
277-
typed-ast==1.4.1
286+
typed-ast==1.2.0
278287
typing==3.7.4.1
279288
typing-extensions==3.7.4.1
280289
ujson==1.35
@@ -283,12 +292,12 @@ urllib3==1.25.8
283292
uvloop==0.14.0
284293
vecstack==0.4.0
285294
versioneer==0.18
286-
vtk
295+
vtk==9.0.1
287296
wcwidth==0.1.8
288297
webencodings==0.5.1
289298
Werkzeug==1.0.0
290299
widgetsnbextension==3.5.1
291-
wrapt==1.11.2
300+
wrapt==1.12.1
292301
yapf==0.29.0
293302
yarl==1.4.2
294303
zict==1.0.0

‎vendor/ngc-pytorch/Dockerfile.20.12-py3

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -204,6 +204,7 @@ RUN ldconfig /usr/local/cuda/targets/x86_64-linux/lib/stubs && \
204204
RUN python3 -m pip install --no-cache-dir \
205205
mpi4py==3.0.3 \
206206
nni==1.9 \
207+
mlflow==1.13 \
207208
scikit-nni==0.2.1
208209

209210
RUN jupyter nbextensions_configurator enable && \
@@ -246,8 +247,7 @@ LABEL ai.backend.kernelspec="1" \
246247
ai.backend.base-distro="ubuntu16.04" \
247248
ai.backend.runtime-type="python" \
248249
ai.backend.runtime-path="/opt/conda/bin/python3" \
249-
ai.backend.service-ports="ipython:pty:3000,jupyter:http:8080,jupyterlab:http:8090,vscode:http:8180,tensorboard:http:6006"
250-
250+
ai.backend.service-ports="ipython:pty:3000,jupyter:http:8091,jupyterlab:http:8090,vscode:http:8180,tensorboard:http:6006,mlflow-ui:http:5000,nniboard:http:8080"
251251

252252
WORKDIR /home/work
253253
# vim: ft=dockerfile

‎vendor/ngc-pytorch/custom.css

Lines changed: 0 additions & 13 deletions
This file was deleted.

0 commit comments

Comments
 (0)