base kernel for Python 3.6 & CUDA 10.0

Author: hephaex

commons/Dockerfile.base.19.09-py36-cuda10

@@ -1,58 +1,113 @@
 FROM nvidia/cuda:10.0-devel-ubuntu18.04
 
 # TensorFlow version is tightly coupled to CUDA and cuDNN so it should be selected carefully
-ENV TENSORFLOW_VERSION=1.14.0
-ENV PYTORCH_VERSION=1.4.0
-ENV TORCHVISION_VERSION=0.5.0
-ENV TENSORBOARDX_VERSION=1.9
-ENV MXNET_VERSION=1.5.1
-ENV CUDNN_VERSION=7.6.0.64-1+cuda10.0
+ARG CUDA=10.0
+ARG LIBNVINFER=6.0.1-1
+ARG LIBNVINFER_MAJOR_VERSION=6
+ARG CUDNN=7.6.2.24-1
 ENV NCCL_VERSION=2.4.8-1+cuda10.0
+ENV DEBIAN_FRONTEND=noninteractive
+ENV mecab_dicdir /usr/local/lib/mecab/dic/mecab-ko-dic
 
 # Python 2.7 or 3.6 is supported by Ubuntu Bionic out of the box
 ARG python=3.6
 ENV PYTHON_VERSION=${python}
 
 ENV PYTHONUNBUFFERED=1 \
-    LD_LIBRARY_PATH="/usr/local/cuda/lib64:/usr/local/nvidia/lib64" \
-    PATH="/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \
+    LD_LIBRARY_PATH="/usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64:/usr/local/nvidia/lib64:/usr/local/cuda/lib64/stubs:/usr/include/x86_64-linux-gnu" \
+    PATH="/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin/mecab" \
     LANG=C.UTF-8
 
 # Set default shell to /bin/bash
 SHELL ["/bin/bash", "-cu"]
 
-RUN apt-get update && apt-get install -y --allow-downgrades --allow-change-held-packages --no-install-recommends \
+RUN apt-get update -y && \
+    apt-get install -y --no-install-recommends software-properties-common && \
+    add-apt-repository -y "deb http://security.ubuntu.com/ubuntu xenial-security main" && \
+    apt-get update -y && \
+    apt-get install -y --allow-downgrades --allow-change-held-packages --no-install-recommends \
         build-essential \
-        cmake \
-        g++-4.8 \
+	ca-certificates \
+        gcc g++ make \
+	gfortran \
 	dkms \
         git \
         curl \
         vim \
-        wget \
+	yasm \
+        wget zip unzip \
         ca-certificates \
         software-properties-common \
   	openssh-client openssh-server \
+        libssl-dev \
+        libmpdec2 \
 	pdsh curl sudo net-tools \
-	vim iputils-ping wget \
-        libcudnn7=${CUDNN_VERSION} \
+	iputils-ping \
+        cuda-command-line-tools-${CUDA/./-} \
+        libcublas10=10.2.1.243-1 \ 
+        libcublas-dev=10.2.1.243-1 \
+        cuda-nvrtc-${CUDA/./-} \
+        cuda-nvrtc-dev-${CUDA/./-} \
+        cuda-cudart-dev-${CUDA/./-} \
+        cuda-cufft-dev-${CUDA/./-} \
+        cuda-curand-dev-${CUDA/./-} \
+        cuda-cusolver-dev-${CUDA/./-} \
+        cuda-cusparse-dev-${CUDA/./-} \
+        libcudnn7=${CUDNN}+cuda${CUDA} \
+	libcudnn7-dev=${CUDNN}+cuda${CUDA} \
         libnccl2=${NCCL_VERSION} \
         libnccl-dev=${NCCL_VERSION} \
+        libcurl3-dev \
+        libfreetype6-dev \
+        libhdf5-serial-dev \
+        libzmq3-dev \
         libjpeg-dev \
         libpng-dev \
 	libsm6 \
 	libxext6 \
 	libxrender-dev \
 	libcairo2-dev libgirepository1.0-dev pkg-config gir1.2-gtk-3.0 \
+        libgeos-dev libgeos++-dev \
+        pkg-config \
+        zlib1g-dev \
+	mime-support \
 	proj-bin libproj-dev libgeos-dev libgeos++-dev graphviz \
         python${PYTHON_VERSION} \
         python${PYTHON_VERSION}-dev \
-	libnuma-dev
+	libnuma-dev \
+	libjasper-dev \
+        libtiff-dev \
+	libavcodec-dev \
+        libavformat-dev \
+	libswscale-dev \
+	libxine2-dev \
+	libv4l-dev \
+	libboost-dev \
+	libboost-system-dev \
+	libboost-filesystem-dev \
+	fonts-nanum \
+	fonts-nanum-coding \
+	fonts-nanum-extra \
+	&& \
+    find /usr/local/cuda-${CUDA}/lib64/ -type f -name 'lib*_static.a' -not -name 'libcudart_static.a' -delete && \
+    rm /usr/lib/x86_64-linux-gnu/libcudnn_static_v7.a
+
+RUN ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1 \
+    && echo "/usr/local/cuda/lib64/stubs" > /etc/ld.so.conf.d/z-cuda-stubs.conf \
+    && ldconfig
 
 RUN curl -sL https://deb.nodesource.com/setup_10.x | bash - && \
     apt-get update -y && \
     apt-get install -y nodejs
 
+RUN apt-get update && \
+        apt-get install -y --no-install-recommends libnvinfer${LIBNVINFER_MAJOR_VERSION}=${LIBNVINFER}+cuda${CUDA} \
+        libnvinfer-dev=${LIBNVINFER}+cuda${CUDA} \
+        libnvinfer-plugin-dev=${LIBNVINFER}+cuda${CUDA} \
+        libnvinfer-plugin${LIBNVINFER_MAJOR_VERSION}=${LIBNVINFER}+cuda${CUDA} \
+        && apt-get clean \
+        && rm -rf /var/lib/apt/lists/*
+
 # OFED
 ENV STAGE_DIR=/tmp
 RUN mkdir -p ${STAGE_DIR}
@@ -81,7 +136,7 @@ RUN ln -s /usr/local/cuda-10.0 /usr/local/cuda && \
     ldconfig
 
 RUN if [[ "${PYTHON_VERSION}" == "3.6" ]]; then \
-        apt-get install -y python${PYTHON_VERSION}-distutils python-apt; \
+        apt-get install -y python${PYTHON_VERSION}-distutils python-apt ; \
     fi
 
 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 2
@@ -93,45 +148,150 @@ RUN curl https://bootstrap.pypa.io/get-pip.py | python3 && \
 # Install TensorFlow, Keras, PyTorch and MXNet
 RUN python3 -m pip install pip --no-cache-dir -Iv \
     	    Cython==0.29.13 \
-            numpy==1.16.5 \
-    	    scipy==1.3.1 \
+	    numpy==1.16.3 \
+    	    scipy==1.4.1 \
 	    scikit-image==0.16.2 \
-    	    scikit-learn==0.21.3 \
-	    matplotlib==3.1.3 \
-      	    tensorflow-gpu==${TENSORFLOW_VERSION}
-
-RUN python3 -m pip install --no-cache-dir \
-    	https://download.pytorch.org/whl/cu100/torch-${PYTORCH_VERSION}%2Bcu100-cp36-cp36m-linux_x86_64.whl \
-        https://download.pytorch.org/whl/cu100/torchvision-${TORCHVISION_VERSION}%2Bcu100-cp36-cp36m-linux_x86_64.whl && \
-    python3 -m pip install --no-cache-dir tensorboardX==${TENSORBOARDX_VERSION}	
-RUN python3 -m pip install --no-cache-dir mxnet-cu100==${MXNET_VERSION}
-RUN python3 -m pip install --no-cache-dir --extra-index-url \
-    	    https://developer.download.nvidia.com/compute/redist/cuda/10.0 \
-	    nvidia-dali \
-	    nvidia-dali-tf-plugin 
+    	    scikit-learn==0.22.2 \
+	    tornado==6.0.4 \
+	    pystan==2.19.1.1 \
+	    pycairo==1.19.0 \
+	    matplotlib==3.1.3 
+RUN python3 -m pip install pip --no-cache-dir \
+    	    Cartopy==0.17.0 \
+	    notebook==6.0.3 
 
 WORKDIR /tmp
 COPY ./requirements.19.09.txt /tmp
 RUN python3 -m pip install --no-cache-dir -Ir requirements.19.09.txt
 
-RUN python3 -m pip install --no-cache-dir -Iv \
-    	    tensorwatch==0.8.10 \
-	    && \
-    rm -rf /root/.cache && \
+# NLP
+RUN python3 -m pip install --no-cache-dir konlpy h5py && \
     rm -f /tmp/*.whl /tmp/requirements.19.09.txt
 
-# Jupyter notebook extension
-RUN mkdir -p /home/work/.jupyter/nbextension
-WORKDIR /home/work/.jupyter/nbextension
+RUN apt-get update && \
+    apt-get install -y \
+    	openjdk-8-jdk \
+	automake \
+	&& \
+    cd /tmp && \
+    curl -LO https://bitbucket.org/eunjeon/mecab-ko/downloads/mecab-0.996-ko-0.9.2.tar.gz && \
+    tar zxfv mecab-0.996-ko-0.9.2.tar.gz && \
+    cd mecab-0.996-ko-0.9.2 && \
+    ./configure && \
+    make -j$(nproc) && \
+    make check && \
+    make install
+
+RUN echo "Install mecab-ko-dic" && \
+    cd /tmp && \
+    ldconfig && \
+    curl -LO https://bitbucket.org/eunjeon/mecab-ko-dic/downloads/mecab-ko-dic-2.1.1-20180720.tar.gz && \
+    tar -zxvf mecab-ko-dic-2.1.1-20180720.tar.gz && \
+    cd mecab-ko-dic-2.1.1-20180720 && \
+    ./autogen.sh && \
+    ./configure && \
+    make -j$(nproc) && \
+    sh -c 'echo "dicdir=/usr/local/lib/mecab/dic/mecab-ko-dic" > /usr/local/etc/mecabrc' && \
+    make install && \
+    cd /tmp && \
+    git clone https://bitbucket.org/eunjeon/mecab-python-0.996.git && \
+    python3 -m pip install /tmp/mecab-python-0.996
+
+# OpenCV
+RUN ln -s /usr/include/libv4l1-videodev.h /usr/include/linux/videodev.h && \
+   apt-get install -y \
+	libgstreamer1.0-dev \
+	libgstreamer-plugins-base1.0-dev \
+	libgtk-3-dev \
+        libtbb-dev \
+	libatlas-base-dev \
+	libdc1394-22-dev \
+	libxvidcore-dev \
+	libfaac-dev \
+	libmp3lame-dev \
+	libtheora-dev \
+	libvorbis-dev \
+	libxvidcore-dev \
+	libopencore-amrnb-dev libopencore-amrwb-dev \
+	libavresample-dev \
+	x264 \
+	libx264-dev \
+	v4l-utils \	
+	libprotobuf-dev protobuf-compiler \
+	libgoogle-glog-dev libgflags-dev \
+	libgphoto2-dev \
+	libeigen3-dev \
+	libhdf5-dev \	
+    && \	
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/
+
+RUN wget https://github.com/Kitware/CMake/releases/download/v3.15.3/cmake-3.15.3-Linux-x86_64.sh \
+         -q -O /tmp/cmake-install.sh && \
+    chmod u+x /tmp/cmake-install.sh && \
+    mkdir /usr/bin/cmake && \
+    /tmp/cmake-install.sh --skip-license --prefix=/usr/bin/cmake && \
+    rm /tmp/cmake-install.sh
 
-RUN jupyter nbextensions_configurator enable && \
-    jupyter contrib nbextension install && \
-    jupyter nbextension enable --py --sys-prefix widgetsnbextension && \
-    jupyter contrib nbextension install && \
-    jupyter serverextension enable --py jupyterlab --sys-prefix && \
-    jupyter labextension install @jupyter-widgets/jupyterlab-manager && \
-    git clone https://github.com/lambdalisue/jupyter-vim-binding vim_binding && \
-    jupyter nbextension enable /home/work/.jupyter/nbextension/vim_binding/vim_binding
+ENV PATH="/usr/bin/cmake/bin:${PATH}"
+
+WORKDIR /tmp
+ENV OPENCV_VERSION="4.1.0"
+RUN wget https://github.com/opencv/opencv/archive/${OPENCV_VERSION}.zip && \
+    wget -O opencv-contrib.zip https://github.com/opencv/opencv_contrib/archive/${OPENCV_VERSION}.zip && \
+    unzip ${OPENCV_VERSION}.zip && \
+    unzip opencv-contrib.zip && \
+    mkdir opencv-${OPENCV_VERSION}/cmake_binary && \
+    cd opencv-${OPENCV_VERSION}/cmake_binary && \
+    cmake \
+      -DCMAKE_BUILD_TYPE=RELEASE \
+      -D BUILD_TIFF=ON \
+      -D BUILD_opencv_java=OFF \
+      -D WITH_CUDA=ON \
+      -D CUDA_NVCC_FLAGS=--expt-relaxed-constexpr \
+      -D CUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda-10.0 \
+      -D ENABLE_FAST_MATH=1 \
+      -D CUDA_FAST_MATH=1 \
+      -D WITH_CUBLAS=1 \
+      -D WITH_OPENGL=ON \
+      -D WITH_OPENCL=ON \
+      -D WITH_IPP=ON \
+      -D WITH_TBB=ON \
+      -D WITH_EIGEN=ON \
+      -D WITH_V4L=ON \
+      -D BUILD_TESTS=OFF \
+      -D BUILD_PERF_TESTS=OFF \
+      -D OPENCV_EXTRA_MODULES_PATH="../../opencv_contrib-4.1.0/modules" \
+      -D CMAKE_BUILD_TYPE=RELEASE \
+      -D CMAKE_INSTALL_PREFIX=$(python3 -c "import sys; print(sys.prefix)") \
+      -D PYTHON_EXECUTABLE=$(which python3) \
+      -D PYTHON_INCLUDE_DIR=$(python3 -c "from distutils.sysconfig import get_python_inc; print(get_python_inc())") \
+      -D PYTHON_PACKAGES_PATH=$(python3 -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())") \
+      .. 2>&1 | tee cmake_messages.txt && \
+    make -j$(nproc) && \
+    make install && \
+    cd /tmp && \
+    python3 -m pip install --no-cache-dir opencv-python && \
+    rm -fr opencv* 
+
+# XGBoost
+RUN git clone  --recursive https://github.com/dmlc/xgboost && \
+    cd xgboost && \
+    mkdir build && \
+    cd build && \
+    cmake .. -DUSE_CUDA=ON -DUSE_NCCL=ON && \
+    make -j$(nproc) && \
+    cd ../python-package && \
+    python3 setup.py install 
+
+# LightGBM
+RUN git clone --recursive https://github.com/Microsoft/LightGBM && \
+    cd LightGBM && \
+    mkdir build ; cd build && \
+    cmake -DUSE_GPU=1 -DOpenCL_LIBRARY=/usr/local/cuda-10.0/lib64/libOpenCL.so -DOpenCL_INCLUDE_DIR=/usr/local/cuda-10.0/include/ .. && \
+    make -j$(nproc) && \
+    cd ../python-package && \
+    python3 setup.py install --precompile
 
 # Install Open MPI
 RUN mkdir /tmp/openmpi && \
@@ -143,19 +303,8 @@ RUN mkdir /tmp/openmpi && \
     make -j $(nproc) all && \
     make install && \
     ldconfig && \
-    rm -rf /tmp/openmpi
+    rm -rf /tmp/openmpi*
 
-# Install Horovod, temporarily using CUDA stubs
-RUN ldconfig /usr/local/cuda/targets/x86_64-linux/lib/stubs && \
-    HOROVOD_GPU_ALLREDUCE=NCCL HOROVOD_GPU_BROADCAST=NCCL \
-    HOROVOD_WITH_TENSORFLOW=1 HOROVOD_WITH_PYTORCH=1 HOROVOD_WITH_MXNET=1 \
-    pip install --no-cache-dir horovod==0.19.0 && \
-    ldconfig
-
-RUN python3 -m pip install --no-cache-dir \
-    	    mpi4py==3.0.3 \
-	    nni
-	    
 # Install OpenSSH for MPI to communicate between containers
 RUN apt-get install -y --no-install-recommends openssh-client openssh-server && \
     mkdir -p /var/run/sshd
@@ -165,4 +314,9 @@ RUN cat /etc/ssh/ssh_config | grep -v StrictHostKeyChecking > /etc/ssh/ssh_confi
     echo "    StrictHostKeyChecking no" >> /etc/ssh/ssh_config.new && \
     mv /etc/ssh/ssh_config.new /etc/ssh/ssh_config
 
-WORKDIR /home/work
+RUN apt autoclean && \
+    rm -rf /var/lib/apt/lists/* && \	
+    rm -rf /root/.cache && \
+    rm -rf /tmp/*
+
+WORKDIR /home/work