add pybind11

Author: hephaex

commons/Dockerfile.base.py38-cuda11.1

@@ -2,18 +2,18 @@ FROM nvidia/cuda:11.1-cudnn8-devel-ubuntu18.04
 
 # TensorFlow version is tightly coupled to CUDA and cuDNN so it should be selected carefully
 ARG CUDA=11.1
-ARG LIBNVINFER=7.2.2-1
+ARG LIBNVINFER=7.2.3-1
 ARG LIBNVINFER_MAJOR_VERSION=7
 ARG CUDNN=8.0.5.39-1
-ENV NCCL=2.8.3
-ENV NCCL_VERSION=2.8.3-1+cuda11.1
+ENV NCCL=2.8.4
+ENV NCCL_VERSION=2.8.4-1+cuda11.1
 ENV CUDNN_VERSION 8.0.5.43
 ENV MLNX_OFED_VERSION=5.1-2.3.7.1
 ENV MKL_VERSION=2020.3
 ENV MKL_BUILD=279
-ENV NUMPY_VERSION 1.19.4
-ENV SCIPY_VERSION 1.5.4
-ENV OPENMPI 4.0.5
+ENV NUMPY_VERSION 1.20.2
+ENV SCIPY_VERSION 1.6.2
+ENV OPENMPI 4.1.0
 # Python 3.8 is supported by Ubuntu Bionic out of the box
 ARG python=3.8
 ENV PYTHON_VERSION=${python}
@@ -45,71 +45,79 @@ RUN rm -fr /var/lib/apt/lists/* && \
     wget https://apt.repos.intel.com/setup/intelproducts.list -O /etc/apt/sources.list.d/intelproducts.list && \
     apt-get update -y && \
     apt-get install -y --allow-downgrades --allow-change-held-packages --no-install-recommends \
-        build-essential \
+	bashtop \
 	ca-certificates \
-        gcc g++ make \
-	gfortran \
+	curl \
 	dkms \
-        git subversion \
-        vim \
-	htop bashtop \
-	yasm \
-        wget zip unzip \
-  	openssh-client openssh-server \
-        libssl-dev \
-        libmpdec2 \
-	pdsh curl net-tools \
+	fonts-nanum \
+	fonts-nanum-coding \
+	fonts-nanum-extra \
+	g++ \
+	gfortran \
+	htop \
+	intel-mkl-2020.3-111 intel-ipp-2020.2-108 intel-tbb-2020.3-108 intel-daal-2020.2-108 intel-mpi-2019.8-108 \
 	iputils-ping \
+	libasound2-dev \
+	libavcodec-dev \
+	libboost-dev \
+	libboost-filesystem-dev \
+	libboost-system-dev \
+	libcairo2-dev libgirepository1.0-dev pkg-config gir1.2-gtk-3.0 \
+	libgdal-dev python3-gdal \
+	libjasper-dev \
+	libnuma-dev \
+	libopenblas-dev liblapack-dev \
+	libsm6 \
+	libswscale-dev \
+	libv4l-dev \
+	libxext6 \
+	libxine2-dev \
+	libxml2-dev \
+	libxrender-dev \
+	libxslt1-dev \
+	make \
+	mime-support \
+	net-tools \
+	openssh-server \
+	pdsh \
+	proj-bin libproj-dev libgeos-dev libgeos++-dev graphviz \
+	subversion \
+	unzip \
+	xvfb \
+	xz-utils \
+	yasm \
+	zip \
+  	openssh-client \
+        build-essential \
+        gcc \
+        git \
+        libavformat-dev \
         libcurl3-dev \
         libfreetype6-dev \
+        libgeos-dev libgeos++-dev \
         libhdf5-serial-dev \
-        libzmq3-dev \
         libjpeg-dev \
+        libmpdec2 \
         libpng-dev \
-	libsm6 \
-	libxext6 \
-	libxrender-dev \
-	libgdal-dev python3-gdal \
         libsdl-dev libsdl-image1.2-dev libsdl-mixer1.2-dev libsdl-ttf2.0-dev \
         libsmpeg-dev libportmidi-dev libavformat-dev libswscale-dev \
-	libcairo2-dev libgirepository1.0-dev pkg-config gir1.2-gtk-3.0 \
-        libgeos-dev libgeos++-dev \
+        libssl-dev \
+        libtiff-dev \
+        libzmq3-dev \
         pkg-config \
-        zlib1g-dev \
-	mime-support \
-	intel-mkl-2020.3-111 intel-ipp-2020.2-108 intel-tbb-2020.3-108 intel-daal-2020.2-108 intel-mpi-2019.8-108 \
-	proj-bin libproj-dev libgeos-dev libgeos++-dev graphviz \
         python${PYTHON_VERSION} \
         python${PYTHON_VERSION}-dev \
-	libxml2-dev \
-	libxslt1-dev \
-	libasound2-dev \
-	libnuma-dev \
-	libjasper-dev \
-        libtiff-dev \
-	libavcodec-dev \
-        libavformat-dev \
-	libswscale-dev \
-	libxine2-dev \
-	libv4l-dev \
-	libboost-dev \
-	libboost-system-dev \
-	libboost-filesystem-dev \
-	libopenblas-dev liblapack-dev \
-	xvfb \
-	xz-utils \
-	fonts-nanum \
-	fonts-nanum-coding \
-	fonts-nanum-extra \
-	&& \
+        vim \
+        wget \
+        zlib1g-dev && \
     echo "/opt/intel/daal/lib/intel64" >> /etc/ls.so.conf && \
     echo "/opt/intel/ipp/lib/intel64" >> /etc/ls.so.conf && \
     echo "/opt/intel/lib/intel64" >> /etc/ls.so.conf && \
     echo "/opt/intel/mkl/lib/intel64" >> /etc/ls.so.conf && \
     find /usr/local/cuda-${CUDA}/lib64/ -type f -name 'lib*_static.a' -not -name 'libcudart_static.a' -delete
 
 WORKDIR /tmp
-RUN curl -fsSL https://developer.download.nvidia.com/compute/redist/nccl/v2.8/nccl_2.8.3-1+cuda11.1_x86_64.txz -O && \
+RUN curl -fsSL https://developer.download.nvidia.com/compute/redist/nccl/v2.8/nccl_2.8.4-1+cuda11.1_x86_64.txz -O && \
     tar --no-same-owner --keep-old-files --lzma -xvf nccl_${NCCL}-1+cuda11.1_x86_64.txz -C /usr/local/cuda/lib64/ --strip-components=2 --wildcards '*/lib/libnccl.so.*' && \
     tar --no-same-owner --keep-old-files --lzma -xvf  nccl_${NCCL}-1+cuda11.1_x86_64.txz -C /usr/lib/pkgconfig/ --strip-components=3 --wildcards '*/lib/pkgconfig/*' && \
     rm nccl_${NCCL}-1+cuda11.1_x86_64.txz && \
@@ -124,7 +132,7 @@ RUN ln -s /usr/local/cuda-11.1 /usr/local/cuda && \
     ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so.8.0.5 /usr/local/cuda/lib64/libcudnn.so && \
     ldconfig
 
-RUN curl -sL https://deb.nodesource.com/setup_10.x | bash - && \
+RUN curl -sL https://deb.nodesource.com/setup_12.x | bash - && \
     apt-get update -y && \
     apt-get install -y nodejs
 
@@ -187,7 +195,8 @@ RUN rm /usr/bin/python /usr/bin/python3 && \
 
 WORKDIR /tmp
 RUN curl https://bootstrap.pypa.io/get-pip.py | python3 && \
-    python3 -m pip install --no-cache-dir -U setuptools pip
+    python3 -m pip install --no-cache-dir -U setuptools pip && \
+    python3 -m pip install --no-cache-dir cython pybind11
 
 WORKDIR /tmp
 # OpenBlas
@@ -288,7 +297,7 @@ RUN ln -s /usr/include/libv4l1-videodev.h /usr/include/linux/videodev.h && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/
 
-RUN wget https://github.com/Kitware/CMake/releases/download/v3.19.2/cmake-3.19.2-Linux-x86_64.sh \
+RUN wget https://github.com/Kitware/CMake/releases/download/v3.20.0/cmake-3.20.0-Linux-x86_64.sh \
          -q -O /tmp/cmake-install.sh && \
     chmod u+x /tmp/cmake-install.sh && \
     mkdir /usr/bin/cmake && \
@@ -352,9 +361,9 @@ RUN cd /usr/local/src && mkdir lightgbm && cd lightgbm && \
 
 # Install Open UCX
 WORKDIR /tmp
-RUN wget https://github.com/openucx/ucx/archive/v1.9.0.tar.gz && \
-    tar xvf v1.9.0.tar.gz && \
-    cd ucx-1.9.0 && \
+RUN wget https://github.com/openucx/ucx/archive/v1.10.0.tar.gz && \
+    tar xvf v1.10.0.tar.gz && \
+    cd ucx-1.10.0 && \
     bash ./autogen.sh && \
     mkdir build && \
     cd build && \
@@ -375,11 +384,10 @@ RUN mkdir /tmp/openmpi && \
     rm -rf /tmp/openmpi*
 
 # Install OpenSSH for MPI to communicate between containers
-RUN apt-get install -y --no-install-recommends openssh-client openssh-server && \
-    mkdir -p /var/run/sshd
 
 # Allow OpenSSH to talk to containers without asking for confirmation
-RUN cat /etc/ssh/ssh_config | grep -v StrictHostKeyChecking > /etc/ssh/ssh_config.new && \
+RUN mkdir -p /var/run/sshd && \
+    cat /etc/ssh/ssh_config | grep -v StrictHostKeyChecking > /etc/ssh/ssh_config.new && \
     echo "    StrictHostKeyChecking no" >> /etc/ssh/ssh_config.new && \
     mv /etc/ssh/ssh_config.new /etc/ssh/ssh_config
 
@@ -392,26 +400,18 @@ RUN mv /usr/local/bin/mpirun /usr/local/bin/mpirun.real && \
 # Configure OpenMPI to run good defaults:
 RUN echo "btl_tcp_if_exclude = lo,docker0" >> /usr/local/etc/openmpi-mca-params.conf
 
-# Install OpenSSH for MPI to communicate between containers
-RUN mkdir -p /var/run/sshd
-
-# Allow OpenSSH to talk to containers without asking for confirmation
-RUN cat /etc/ssh/ssh_config | grep -v StrictHostKeyChecking > /etc/ssh/ssh_config.new && \
-    echo "    StrictHostKeyChecking no" >> /etc/ssh/ssh_config.new && \
-    mv /etc/ssh/ssh_config.new /etc/ssh/ssh_config
-
 # Install git-lfs
 WORKDIR /tmp
-RUN curl -sLO https://github.com/git-lfs/git-lfs/releases/download/v2.13.1/git-lfs-linux-amd64-v2.13.1.tar.gz && \
-    tar -zxf git-lfs-linux-amd64-v2.13.1.tar.gz && \
+RUN curl -sLO https://github.com/git-lfs/git-lfs/releases/download/v2.13.3/git-lfs-linux-amd64-v2.13.3.tar.gz && \
+    tar -zxf git-lfs-linux-amd64-v2.13.3.tar.gz && \
     bash install.sh && \
     rm -rf /tmp/*
 
 # Install VSCode
-RUN curl -fL https://github.com/cdr/code-server/releases/download/v3.8.0/code-server-3.8.0-linux-amd64.tar.gz \
+RUN curl -fL https://github.com/cdr/code-server/releases/download/v3.9.2/code-server-3.9.2-linux-amd64.tar.gz \
   | tar -C /usr/local/lib -xz && \
-    mv /usr/local/lib/code-server-3.8.0-linux-amd64 /usr/local/lib/code-server-3.8.0 && \
-    ln -s /usr/local/lib/code-server-3.8.0/bin/code-server /usr/local/bin/code-server
+    mv /usr/local/lib/code-server-3.9.2-linux-amd64 /usr/local/lib/code-server-3.9.2 && \
+    ln -s /usr/local/lib/code-server-3.9.2/bin/code-server /usr/local/bin/code-server
 
 # Install Jupyterlab extensions
 RUN jupyter nbextensions_configurator enable && \