Merge branch '20.12' of https://github.com/lablup/backend.ai-kernels into 20.12

Author: hephaex

commons/Dockerfile.base.cuda11.1

@@ -18,10 +18,10 @@ ENV OPENMPI 4.0.5
 ENV DEBIAN_FRONTEND=noninteractive \
     MPLBACKEND=Svg \
     PYTHONUNBUFFERED=1 \
-    LIBRARY_PATH=/usr/local/cuda/lib64/stubs \
+    LIBRARY_PATH="/usr/local/cuda/lib64/stubs" \
     _CUDA_COMPAT_PATH="/usr/local/cuda/compat" \
-    LD_LIBRARY_PATH="/usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/compat/lib:/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda/lib64:/usr/local/nvidia/lib64:/usr/include/x86_64-linux-gnu" \
-    PATH="/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/bin/cmake/bin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/src/lightgbm/LightGBM:/usr/local/bin/mecab" \
+    LD_LIBRARY_PATH="/usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/compat/lib:/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda/lib64:/usr/include/x86_64-linux-gnu:/opt/intel/compilers_and_libraries_2020.3.279/linux/mkl/lib/intel64" \
+    PATH="/usr/local/mpi/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/bin/cmake/bin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/tensorrt/bin:/usr/local/src/lightgbm/LightGBM:/usr/local/bin/mecab" \
     mecab_dicdir=/usr/local/lib/mecab/dic/mecab-ko-dic \
     LANG=C.UTF-8
 
@@ -101,8 +101,6 @@ RUN rm -fr /var/lib/apt/lists/* && \
     echo "/opt/intel/mkl/lib/intel64" >> /etc/ls.so.conf && \
     find /usr/local/cuda-${CUDA}/lib64/ -type f -name 'lib*_static.a' -not -name 'libcudart_static.a' -delete
 
-# NCCL_DOWNLOAD_SUM=34000cbe6a0118bfd4ad898ebc5f59bf5d532bbf2453793891fa3f1621e25653
-# echo "$NCCL_DOWNLOAD_SUM  nccl_2.8.3-1+cuda11.1_x86_64.txz" | sha256sum -c - 
 RUN curl -fsSL https://developer.download.nvidia.com/compute/redist/nccl/v2.8/nccl_2.8.3-1+cuda11.1_x86_64.txz -O && \
     tar --no-same-owner --keep-old-files --lzma -xvf nccl_${NCCL}-1+cuda11.1_x86_64.txz -C /usr/local/cuda/lib64/ --strip-components=2 --wildcards '*/lib/libnccl.so.*' && \
     tar --no-same-owner --keep-old-files --lzma -xvf  nccl_${NCCL}-1+cuda11.1_x86_64.txz -C /usr/lib/pkgconfig/ --strip-components=3 --wildcards '*/lib/pkgconfig/*' && \
@@ -125,21 +123,13 @@ RUN curl -sL https://deb.nodesource.com/setup_10.x | bash - && \
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
 	libnvinfer${LIBNVINFER_MAJOR_VERSION}=${LIBNVINFER}+cuda${CUDA} \
-#        libnvinfer-bin=${LIBNVINFER}+cuda${CUDA} \
         libnvinfer-dev=${LIBNVINFER}+cuda${CUDA} \
 	libnvinfer-plugin7 \
         libnvinfer-plugin-dev=${LIBNVINFER}+cuda${CUDA} \
         && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
-WORKDIR /tmp
-#RUN mkdir -p /opt/intel/lib && \
-#    curl -fsSL https://anaconda.org/intel/mkl-static/${MKL_VERSION}/download/linux-64/mkl-static-${MKL_VERSION}-intel_${MKL_BUILD}.tar.bz2 | tar xjv && \
-#    mv lib/* /opt/intel/lib/ && \
-#    curl -fsSL https://anaconda.org/intel/mkl-include/${MKL_VERSION}/download/linux-64/mkl-include-${MKL_VERSION}-intel_${MKL_BUILD}.tar.bz2 | tar xjv && \
-#    mv include /opt/intel/
-
 # Add OpenCL ICD files for LightGBM
 RUN mkdir -p /etc/OpenCL/vendors && \
     echo "libnvidia-opencl.so.1" > /etc/OpenCL/vendors/nvidia.icd
@@ -268,8 +258,7 @@ RUN python3 -m pip install pip --no-cache-dir \
 WORKDIR /tmp
 COPY ./requirements.txt /tmp
 RUN python3 -m pip install --no-cache-dir -r requirements.txt && \
-    rm -f /tmp/*.whl /tmp/requirements.txt
-
+    rm -f /tmp/requirements.txt
 
 # OpenCV
 RUN ln -s /usr/include/libv4l1-videodev.h /usr/include/linux/videodev.h && \
@@ -341,10 +330,10 @@ RUN wget https://github.com/opencv/opencv/archive/${OPENCV_VERSION}.zip && \
       -D PYTHON_PACKAGES_PATH=$(python3 -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())") \
       .. 2>&1 | tee cmake_messages.txt && \
     make -j$(nproc) && \
-    make install && \
-    cd /tmp && \
-    python3 -m pip install --no-cache-dir opencv-python && \
-    rm -fr opencv* 
+    make install 
+#RUN cd /tmp && \
+#    python3 -m pip install --no-cache-dir ./opencv-contrib-python*.whl && \
+#    rm -fr opencv* 
 
 # XGBoost
 RUN git clone  --recursive https://github.com/dmlc/xgboost && \
@@ -365,9 +354,8 @@ RUN cd /usr/local/src && mkdir lightgbm && cd lightgbm && \
     cd /usr/local/src/lightgbm/LightGBM/python-package && \
     python3 setup.py install --precompile
 
- 
 # Install Open MPI
-RUN mkdir /tmp/openmpi && \
+RUN mkdir -p /tmp/openmpi && \
     cd /tmp/openmpi && \
     wget https://www.open-mpi.org/software/ompi/v4.0/downloads/openmpi-${OPENMPI}.tar.gz && \
     tar zxf openmpi-${OPENMPI}.tar.gz && \
@@ -422,7 +410,7 @@ RUN jupyter nbextensions_configurator enable && \
     jupyter nbextension enable --py --sys-prefix widgetsnbextension && \
     jupyter serverextension enable --py jupyterlab --sys-prefix && \
     jupyter labextension install --no-build @jupyter-widgets/jupyterlab-manager && \
-#    git clone https://github.com/lambdalisue/jupyter-vim-binding vim_binding && \
+    git clone https://github.com/lambdalisue/jupyter-vim-binding vim_binding && \
 #    jupyter nbextension enable /home/work/.jupyter/nbextension/vim_binding/vim_binding && \
 #    jupyter labextension install --no-build @lckr/jupyterlab_variableinspector && \
     jupyter labextension install --no-build @krassowski/jupyterlab-lsp && \
@@ -437,6 +425,7 @@ RUN jupyter nbextensions_configurator enable && \
     jupyter lab build
 
 RUN apt autoclean && \
+    ln -s /usr/local/cuda-11.1/targets/x86_64-linux/lib/libcusolver.so.11 /usr/local/cuda-11.1/targets/x86_64-linux/lib/libcusolver.so.10 && \
     sed -i 's/source \/usr\/local\/nvm\/nvm.sh//' /etc/bash.bashrc && \
     python3 -m pip uninstall -y  tensorboard-plugin-wit && \
     rm -rf /var/lib/apt/lists/* && \	

python-tensorflow/Dockerfile.2.4-py36-cuda11.1

@@ -0,0 +1,94 @@
+FROM lablup/common-base:py36-cuda11.1
+
+RUN python3 -m pip install --no-cache-dir --upgrade \
+	    -q git+https://github.com/google-research/tensorflow_constrained_optimization 
+
+RUN python3 -m pip install --no-cache-dir \
+      --use-feature=2020-resolver \
+        requests_oauthlib==1.2.0 \
+	neural-structured-learning==1.3.1 \
+	pydot==1.4.0 \
+	proto-plus==1.10.2
+RUN python3 -m pip install --no-cache-dir \
+      --use-feature=2020-resolver \
+	    tensorflow-transform==0.25.0 \
+	    tensorflow_model_analysis \
+	    tensorflow-datasets \
+	    tensorflow_probability==0.11.1 \ 
+	    tensorflow-addons==0.11.2 \
+	    neural-structured-learning==1.3.1 \
+	    tensorflow-model-optimization==0.5.0 \
+	    tfx==0.25.0 \
+	    tfx-bsl==0.25.0 \ 
+	    tf-agents==0.6.0  
+RUN python3 -m pip install --no-cache-dir \
+      --use-feature=2020-resolver \
+            tensorflow-cloud==0.1.10 \
+            tensorflow-hub==0.9.0 \
+	    tensorflow_text==2.4.1 \
+	    tensorflow-graphics-gpu==1.0.0 \
+	    tensorflow-gan==2.0.0 \
+	    tensorflow-data-validation \
+	    tensorflow-plot==0.3.2 \
+	    tensorflow-lattice==2.0.7 \
+	    tensorflow_ranking==0.3.2 \
+	    tensorflow-compression==1.3 \
+	    mesh-tensorflow==0.1.13 \
+	    google_cloud_spanner==1.19.0 \
+	    pymongo==3.11.2 \
+	    googleapis-common-protos==1.51.0 \
+	    google_cloud_videointelligence==1.15.0 \
+	    google_cloud_vision==0.42.0 \
+	    google_cloud_pubsub==1.6.0 \
+	    google_cloud_dlp==0.15.0 \
+	    google_cloud_datastore==1.15.2 \
+	    fastavro==1.2.0 \
+	    google_cloud_bigtable==1.6.0 \
+	    google_api_core==1.24.0 \
+	    pbr==5.5.1 \
+	    mock==1.3.0 \
+	    proto-plus==1.13.0 \
+	    oauth2client==4.1.3 \
+	    joblib==0.14.0 \
+	    libcst==0.3.14 \
+            tensorflow-io==0.17.0
+
+RUN python3 -m pip install --extra-index-url \
+       https://developer.download.nvidia.com/compute/redist nvidia-dali-cuda100 \
+       && \
+    python3 -m pip install --extra-index-url \
+       https://developer.download.nvidia.com/compute/redist nvidia-dali-tf-plugin-cuda100
+
+# Install Horovod, temporarily using CUDA stubs
+RUN ldconfig /usr/local/cuda/targets/x86_64-linux/lib/stubs && \
+    HOROVOD_GPU_ALLREDUCE=NCCL HOROVOD_GPU_BROADCAST=NCCL HOROVOD_NCCL_LINK=SHARED \
+    HOROVOD_WITH_TENSORFLOW=1 HOROVOD_WITHOUT_PYTORCH=1 HOROVOD_WITHOUT_MXNET=1\
+    HOROVOD_GPU=CUDA \
+    python3 -m pip install --no-cache-dir horovod==0.21.0 && \
+    ldconfig
+
+RUN python3 -m pip install --no-cache-dir \
+            mpi4py==3.0.3 \
+            nni==1.9 \
+            mlflow==1.12.1 \
+            scikit-nni==0.2.1
+	    
+# Install ipython kernelspec
+Run python3 -m ipykernel install --display-name "TensorFlow 2.4 on Python 3.6 & CUDA 11ᇂᇂᇂᇂ.1" && \
+    cat /usr/local/share/jupyter/kernels/python3/kernel.json
+
+# Backend.AI specifics
+LABEL ai.backend.kernelspec="1" \
+      ai.backend.envs.corecount="OPENBLAS_NUM_THREADS,OMP_NUM_THREADS,NPROC" \
+      ai.backend.features="batch query uid-match user-input" \
+      ai.backend.base-distro="ubuntu16.04" \
+      ai.backend.resource.min.cpu="1" \
+      ai.backend.resource.min.mem="1g" \
+      ai.backend.resource.min.cuda.device=0 \
+      ai.backend.resource.min.cuda.shares=0 \
+      ai.backend.runtime-type="python" \
+      ai.backend.runtime-path="/usr/bin/python3" \
+      ai.backend.service-ports="ipython:pty:3000,jupyter:http:8080,jupyterlab:http:8090,vscode:http:8180,tensorboard:http:6006,mlflow-ui:preopen:5000,nniboard:preopen:8080"
+      
+WORKDIR /home/work
+# vim: ft=dockerfile

vendor/ngc-pytorch/Dockerfile.20.11-py3

@@ -246,7 +246,7 @@ LABEL ai.backend.kernelspec="1" \
       ai.backend.base-distro="ubuntu16.04" \
       ai.backend.runtime-type="python" \
       ai.backend.runtime-path="/opt/conda/bin/python3" \
-      ai.backend.service-ports="ipython:pty:3000,jupyter:http:8091,jupyterlab:http:8090,vscode:http:8180,tensorboard:http:6006,mlflow-ui:http:5000,nniboard:http:8080"
+      ai.backend.service-ports="ipython:pty:3000,jupyter:http:8091,jupyterlab:http:8090,vscode:http:8180,tensorboard:http:6006,mlflow-ui:preopen:5000,nniboard:preopen:8080"
 
 
 WORKDIR /home/work