support scs-python

Author: hephaex

commons/Dockerfile.base.py38-cuda11.2

@@ -15,14 +15,9 @@ ENV DEBIAN_FRONTEND=noninteractive \
     PIP_IGNORE_INSTALLED=0 \
     PYTHONUNBUFFERED=1 \
     LIBRARY_PATH=/usr/local/cuda/lib64/stubs \
-<<<<<<< HEAD
     _CUDA_COMPAT_PATH="/usr/local/cuda/compat" \
     LD_LIBRARY_PATH="/usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/compat/lib:/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda/lib64:/usr/lib/x86_64-linux-gnu:/usr/include/x86_64-linux-gnu:/usr/local/lib" \
-    PATH="/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/mpi/bin:/usr/local/ucx:/opt/conda/bin:/usr/local/sbin:/usr/bin/cmake/bin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/src/lightgbm/LightGBM" \
-=======
-    _CUDA_COMPAT_PATH="/usr/local/cuda/compat" LD_LIBRARY_PATH="/usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/compat/lib:/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda/lib64:/usr/lib/x86_64-linux-gnu:/usr/include/x86_64-linux-gnu:/usr/local/lib" \
-    PATH="/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/mpi/bin:/usr/local/ucx:/usr/local/sbin:/usr/bin/cmake/bin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/src/lightgbm/LightGBM" \
->>>>>>> 3946c9e80914c8ab44e7d1524d83448a031eeaf4
+    PATH="/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/mpi/bin:/usr/local/ucx:/opt/conda/bin:/usr/bin/cmake/bin:/usr/local/sbin:/usr/bin/cmake/bin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/src/lightgbm/LightGBM" \
     CPLUS_INCLUDE_PATH=/usr/include/gdal \
     C_INCLUDE_PATH=/usr/include/gdal \
     mecab_dicdir=/usr/local/lib/mecab/dic/mecab-ko-dic \
@@ -207,7 +202,6 @@ RUN apt-get update -y && \
     /opt/conda/bin/conda install pip && \
     /opt/conda/bin/python -m pip install --no-cache-dir -U "setuptools <58" 
 
-
 # OFED
 WORKDIR /tmp
 RUN wget -q -O - https://www.mellanox.com/downloads/ofed/MLNX_OFED-5.4-2.4.1.3/MLNX_OFED_LINUX-${MLNX_OFED_VERSION}-ubuntu20.04-x86_64.tgz | tar xzf - && \
@@ -288,38 +282,7 @@ RUN cd /usr/local/src && mkdir lightgbm && cd lightgbm && \
     cd /usr/local/src/lightgbm/LightGBM/python-package && \
     python3 setup.py install --precompile
 
-ENV NUMPY_VERSION 1.22.4 
-ENV SCIPY_VERSION 1.8.1
-ENV MKL_VERSION=2020.3 
-ENV MKL_BUILD=279 
-
-# Numpy    
-RUN git clone --branch=v${NUMPY_VERSION} --depth=1 https://github.com/numpy/numpy.git numpy && \
-    cd numpy && \
-    git checkout -b v${NUMPY_VERSION} && \
-    git submodule update --init && \
-    cp site.cfg.example site.cfg && \
-    echo "[mkl]" >> site.cfg && \
-    echo "include_dirs = /opt/2020.3.279/linux/mkl/include/" >> site.cfg && \
-    echo "library_dirs = /opt/intel/compilers_and_libraries_2020.3.279/linux/mkl/lib/intel64" >> site.cfg && \
-    echo "mkl_libs = mkl_rt" >> site.cfg && \
-    echo "lapack_libs =" >> site.cfg && \
-    python3 setup.py bdist_wheel -d /tmp 
-
-# Scipy
-RUN cd /tmp && \
-    git clone --branch=v${SCIPY_VERSION} --depth=1 https://github.com/scipy/scipy.git scipy && \
-    cd scipy && \
-    git checkout -b v${SCIPY_VERSION} && \
-    git submodule update --init && \
-    cp site.cfg.example site.cfg && \
-    echo "[mkl]" >> site.cfg && \
-    echo "include_dirs = /opt/intel/compilers_and_libraries_2020.3.279/linux/mkl/include/" >> site.cfg && \
-    echo "library_dirs = /opt/intel/compilers_and_libraries_2020.3.279/linux/mkl/lib/intel64" >> site.cfg && \
-    echo "mkl_libs = mkl_rt" >> site.cfg && \
-    echo "lapack_libs =" >> site.cfg && \
-    python3 -m pip install -U --no-cache-dir /tmp/numpy-*.whl && \
-    python3 setup.py install 
+# scs-python
 RUN cd /tmp && \
     git clone --recursive https://github.com/bodono/scs-python.git  && \
     cd scs-python && \
@@ -349,11 +312,6 @@ RUN curl -LO https://bitbucket.org/eunjeon/mecab-ko/downloads/mecab-0.996-ko-0.9
     git clone https://bitbucket.org/eunjeon/mecab-python-0.996.git && \
     python3 -m pip install /tmp/mecab-python-0.996
 
-WORKDIR /tmp
-COPY ./requirements.22.06.txt /tmp/requirements.txt 
-RUN python3 -m pip install --no-cache-dir -r requirements.txt && \
-    rm -f /tmp/*.whl /tmp/requirements.txt
-
 # OpenCV
 WORKDIR /tmp
 ENV OPENCV_VERSION="4.5.5"
@@ -427,10 +385,13 @@ RUN mkdir /tmp/openmpi && \
     echo "btl_tcp_if_exclude = lo,docker0" >> /usr/local/etc/openmpi-mca-params.conf 
 RUN MPICC=/usr/lib64/openmpi/bin/mpicc python3 -m pip install mpi4py && \
     python3 -m pip install "jax[cuda11_cudnn805]" -f https://storage.googleapis.com/jax-releases/jax_releases.html 
-RUN CUDA_ROOT=/usr/local/cuda python3 -m pip install mpi4jax 
-#            nni==1.9 \
-#            mlflow==1.12.1 \
-#            scikit-nni==0.2.1
+
+WORKDIR /tmp
+COPY ./requirements.22.06.txt /tmp/requirements.txt 
+RUN python3 -m pip install --no-cache-dir -r requirements.txt && \
+    rm -f /tmp/*.whl /tmp/requirements.txt
+
+RUN CUDA_ROOT=/usr/local/cuda python3 -m pip install mpi4jax scikit-nni==0.2.1 mlflow==1.12.1 nni==1.9 
 
 RUN ldconfig /usr/local/cuda/targets/x86_64-linux/lib/stubs && \
     HOROVOD_NCCL_LINK=SHARED \