Merge branch 'master' of github.com:lablup/backend.ai-kernels

Author: hephaex

vendor/ngc-tensorflow/Dockerfile.22.02-tf2-py3

@@ -85,7 +85,7 @@ RUN mkdir -p /opt/oracle && \
     ldconfig
 
 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 2 && \
-    python3 -m pip install -U setuptools pip 
+    python3 -m pip install -U "pip < 22" setuptools wheel
 
 # install NLP packages *mecab-ko & khai*
 WORKDIR /tmp
@@ -165,49 +165,6 @@ COPY ./requirements.22.02.txt /tmp/requirements.txt
 RUN python3 -m pip install --no-cache-dir -r requirements.txt && \
     rm -f /tmp/*.whl /tmp/requirements.txt
 
-# install bashtop
-WORKDIR /tmp
-RUN git clone https://github.com/aristocratos/bashtop.git && \
-    cd bashtop && \
-    make install
-
-# install git-lfs
-WORKDIR /tmp
-RUN curl -sLO https://github.com/git-lfs/git-lfs/releases/download/v3.1.2/git-lfs-linux-amd64-v3.1.2.tar.gz && \
-    tar -zxf git-lfs-linux-amd64-v3.1.2.tar.gz && \
-    bash install.sh && \
-    rm -rf /tmp/*
-
-RUN curl -fL https://github.com/cdr/code-server/releases/download/v4.2.0/code-server-4.2.0-linux-amd64.tar.gz | tar -C /usr/local/lib -xz && \
-    mv /usr/local/lib/code-server-4.2.0-linux-amd64 /usr/local/lib/code-server-4.2.0 && \
-    ln -s /usr/local/lib/code-server-4.2.0/bin/code-server /usr/local/bin/code-server
-
-# Install Open MPI
-RUN mkdir /tmp/openmpi && \
-    cd /tmp/openmpi && \
-    wget https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-4.1.2.tar.gz && \
-    tar zxf openmpi-4.1.2.tar.gz && \
-    cd openmpi-4.1.2 && \
-    ./configure --enable-orterun-prefix-by-default && \
-    make -j $(nproc) all && \
-    make install && \
-    ldconfig && \
-    rm -rf /tmp/openmpi*
-# Create a wrapper for OpenMPI to allow running as root by default
-RUN mv /usr/local/bin/mpirun /usr/local/bin/mpirun.real && \
-    echo '#!/bin/bash' > /usr/local/bin/mpirun && \
-    echo 'mpirun.real --allow-run-as-root "$@"' >> /usr/local/bin/mpirun && \
-    chmod a+x /usr/local/bin/mpirun
-
-# Configure OpenMPI to run good defaults:
-RUN echo "btl_tcp_if_exclude = lo,docker0" >> /usr/local/etc/openmpi-mca-params.conf
-
-RUN python3 -m pip install --no-cache-dir \
-    	mpi4py==3.1.3 \
-	    nni==2.6.1 \
-	    mlflow==1.24.0 \
-	    scikit-nni==0.2.1
-
 
 # Install Jupyterlab extensions
 RUN jupyter nbextensions_configurator enable && \
@@ -220,8 +177,7 @@ RUN jupyter nbextensions_configurator enable && \
     jupyter labextension install --no-build @jupyterlab/toc && \
     jupyter nbextension enable execute_time/ExecuteTime && \
     jupyter nbextension enable toc2/main && \
-    jupyter labextension install @jupyterlab/toc && \
-    jupyter lab build    
+    jupyter lab build
 
 RUN apt autoclean && \
     sed -i 's/source \/usr\/local\/nvm\/nvm.sh//' /etc/bash.bashrc && \
@@ -231,7 +187,7 @@ RUN apt autoclean && \
     rm -rf /tmp/*
 
 # Install ipython kernelspec
-RUN /usr/bin/python3 -m ipykernel install --display-name "Python 3.8 (NGC 22.02 / TensorFlow 2.6) on Backend.AI" && \
+RUN /usr/bin/python3 -m ipykernel install --display-name "Python 3.8 (NGC 22.02 / TensorFlow 2.7) on Backend.AI" && \
     cat /usr/local/share/jupyter/kernels/python3/kernel.json
 
 # Backend.AI specifics

vendor/ngc-tensorflow/Dockerfile.22.03-tf2-py3

@@ -68,9 +68,9 @@ RUN curl -sL https://deb.nodesource.com/setup_14.x | bash - && \
 	nodejs \
 	pdsh \
 	protobuf-compiler \
-    libsmpeg-dev \
-    openjdk-8-jdk \
-    python3-gdal \
+        libsmpeg-dev \
+        openjdk-8-jdk \
+        python3-gdal \
 	v4l-utils \	
 	x264 
 
@@ -165,47 +165,6 @@ COPY ./requirements.22.03.txt /tmp/requirements.txt
 RUN python3 -m pip install --no-cache-dir -r requirements.txt && \
     rm -f /tmp/*.whl /tmp/requirements.txt
 
-# install bashtop
-WORKDIR /tmp
-RUN git clone https://github.com/aristocratos/bashtop.git && \
-    cd bashtop && \
-    make install
-
-# install git-lfs
-WORKDIR /tmp
-RUN curl -sLO https://github.com/git-lfs/git-lfs/releases/download/v3.1.2/git-lfs-linux-amd64-v3.1.2.tar.gz && \
-    tar -zxf git-lfs-linux-amd64-v3.1.2.tar.gz && \
-    bash install.sh && \
-    rm -rf /tmp/*
-
-RUN curl -fL https://github.com/cdr/code-server/releases/download/v4.2.0/code-server-4.2.0-linux-amd64.tar.gz | tar -C /usr/local/lib -xz && \
-    mv /usr/local/lib/code-server-4.2.0-linux-amd64 /usr/local/lib/code-server-4.2.0 && \
-    ln -s /usr/local/lib/code-server-4.2.0/bin/code-server /usr/local/bin/code-server
-
-# Install Open MPI
-RUN mkdir /tmp/openmpi && \
-    cd /tmp/openmpi && \
-    wget https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-4.1.2.tar.gz && \
-    tar zxf openmpi-4.1.2.tar.gz && \
-    cd openmpi-4.1.2 && \
-    ./configure --enable-orterun-prefix-by-default && \
-    make -j $(nproc) all && \
-    make install && \
-    ldconfig && \
-    rm -rf /tmp/openmpi* && \
-# Create a wrapper for OpenMPI to allow running as root by default
-    mv /usr/local/bin/mpirun /usr/local/bin/mpirun.real && \
-    echo '#!/bin/bash' > /usr/local/bin/mpirun && \
-    echo 'mpirun.real --allow-run-as-root "$@"' >> /usr/local/bin/mpirun && \
-    chmod a+x /usr/local/bin/mpirun && \
-# Configure OpenMPI to run good defaults:
-    echo "btl_tcp_if_exclude = lo,docker0" >> /usr/local/etc/openmpi-mca-params.conf
-
-RUN python3 -m pip install --no-cache-dir \
-    	mpi4py==3.1.3 \
-	    nni==2.6.1 \
-	    mlflow==1.24.0 \
-	    scikit-nni==0.2.1
 
 # Install Jupyterlab extensions
 RUN jupyter nbextensions_configurator enable && \
@@ -218,8 +177,7 @@ RUN jupyter nbextensions_configurator enable && \
     jupyter labextension install --no-build @jupyterlab/toc && \
     jupyter nbextension enable execute_time/ExecuteTime && \
     jupyter nbextension enable toc2/main && \
-    jupyter labextension install @jupyterlab/toc && \
-    jupyter lab build    
+    jupyter lab build
 
 RUN apt autoclean && \
     sed -i 's/source \/usr\/local\/nvm\/nvm.sh//' /etc/bash.bashrc && \
@@ -234,7 +192,7 @@ RUN /usr/bin/python3 -m ipykernel install --display-name "Python 3.8 (NGC 22.03
 
 # Backend.AI specifics
 COPY ./service-defs /etc/backend.ai/service-defs
-COPY ./runner-scripts/bootstrap.sh runner-scripts/setup_multinode.py /opt/container/
+COPY runner-scripts/bootstrap.sh runner-scripts/setup_multinode.py /opt/container/
 LABEL ai.backend.kernelspec="1" \
       ai.backend.envs.corecount="OPENBLAS_NUM_THREADS,OMP_NUM_THREADS,NPROC" \
       ai.backend.features="batch query uid-match user-input" \

vendor/ngc-tensorflow/requirements.22.02.txt

@@ -3,7 +3,7 @@ aiozmq==0.9.0
 albumentations==1.1.0
 alembic==1.7.5
 ansi2html==1.6.0
-anyio==3.5.0
+anyio
 #apache-airflow[celery,devel,postgres,redis,s3,ssh]==1.10.6
 #apache-beam==2.34.0
 appdirs==1.4.3
@@ -25,7 +25,7 @@ black==19.10b0
 bokeh==2.3.2
 boltons==20.2.1
 boto3==1.17.112
-boto==2.49.0
+boto
 botocore==1.20.112
 branca==0.4.0
 brotli==1.0.9
@@ -47,7 +47,7 @@ convertdate==2.3.2
 coverage==4.5.4
 crcmod==1.7
 cryptography==3.0
-cupy==9.5.0
+cupy==10.2.0
 cvxpy==1.1.15
 cycler==0.10.0
 cython==0.29.21
@@ -74,7 +74,7 @@ docutils==0.15.2
 dopamine_rl==3.0.1
 dtreeviz==0.8.1
 easyProcess==0.3
-easydict==1.9
+easydict
 ecdsa==0.14.1
 ecos==2.0.7.post1
 #elegy==0.8.1
@@ -97,8 +97,8 @@ flask-Compress==1.9.0
 flask==1.1.2
 folium==0.10.0
 funcy==1.13
-gast==0.5.3
-gDAL==3.4.2
+gast==0.4.0
+gDAL==3.0.4
 gensim==3.8.0
 geopandas==0.5.1
 gin_config==0.3.0
@@ -126,7 +126,7 @@ google-resumable-media==1.2.0
 google_api_core==1.23.0
 google_api_python_client==1.12.8
 google_auth_httplib2==0.0.3
-gluonnlp==0.10.0
+gluonnlp
 graphviz==0.14
 greenlet==1.0.0
 grpc-google-iam-v1==0.12.3
@@ -144,7 +144,7 @@ imbalanced-learn==0.5.0
 importlib-metadata~=2.0
 importlib_resources~=1.4
 ipyparallel==6.2.4
-ipywidgets==7.5.1
+ipywidgets==7.6.3
 iso8601==0.1.14
 isort==4.3.21
 itsdangerous==1.1.0
@@ -156,45 +156,28 @@ Jinja2==2.11.3
 json-tricks==3.14.0
 jsonpickle==1.5.2
 jsonschema==4.0.1
-jupyter-client==6.1.12
-jupyter-console==6.2.0
+jupyter-console==6.1.0
 jupyter-contrib-core==0.3.3
 jupyter-contrib-nbextensions==0.5.1
-jupyter-core==4.7.1
 jupyter-dash==0.4.0
 jupyter-highlight-selected-word==0.2.0
 jupyter-js-widgets-nbextension==0.0.2.dev0
-jupyter-kite==1.2.0
 jupyter-latex-envs==1.4.6
-jupyter-lsp==1.5.1
+jupyter-lsp==1.2.0
 jupyter-nbextensions-configurator==0.4.1
-jupyter-packaging==0.9.2
-jupyter-server-mathjax==0.2.2
-jupyter-server-proxy==1.6.0
-jupyter-server==1.15.6
-jupyter-telemetry==0.1.0
+jupyter-server-proxy==1.2.0
 jupyter==1.0.0
 jupyter_bokeh==2.0.4
-jupyterhub-jwtauthenticator==0.1
-jupyterhub-kubespawner==0.16.1
-jupyterhub==1.4.0
-jupyterhubutils==0.25.0
-jupyterlab-code-formatter==1.3.8
-jupyterlab-git==0.24.0
+jupyterlab-git
 jupyterlab-github==2.0.0
-jupyterlab-hdf==0.5.1
-jupyterlab-launcher==0.13.1
 jupyterlab-lsp==3.6.0
-jupyterlab-nvdashboard==0.5.0
-jupyterlab-pygments==0.1.2
-jupyterlab-server==2.10.0
-jupyterlab-translate==1.0.3
-jupyterlab==3.3.2
-#jupyterlabutils==0.10.0
+jupyterlab_hdf==0.3.0
+jupyterlabutils
 jupyterthemes==0.20.0
-jupytext==1.10.1
-keras==2.8.0
-keras-models==0.0.7
+jupytext==1.13.7
+keras==2.7.0
+Keras-Applications==1.0.8
+keras-models
 kiwisolver==1.1.0
 konlpy==0.5.2
 kss==3.3.1.1
@@ -210,6 +193,7 @@ lunardate==0.2.0
 lxml==4.5.0
 mako==1.0.7
 maproxy==0.0.12
+markupsafe==2.0.1
 matplotlib==3.4.3 
 mccabe==0.6.1
 meshio==4.4.2
@@ -225,13 +209,14 @@ msgpack==0.6.1
 multidict==4.7.5
 multiprocess==0.70.9
 munch==2.5.0
-mxnet==1.9.0
+mxnet
 namedlist==1.7
 nbclassic==0.2.7
-nbdime==2.0.0
+nbdime==3.0.0
 netifaces==0.10.9
 networkx==2.4
 numba==0.49.0
+numexpr
 numexpr==2.7.0
 oauth2client==3.0.0
 oauthenticator==0.13.0
@@ -286,7 +271,7 @@ pynput==1.7.3
 #pynvml==11.0.0
 pyproj==2.5.0
 pyqt5==5.15.0
-pyshp==2.2.0
+pyshp
 pystan==3.3.0 
 pytest-cov==2.10.0
 pytest~=6.0
@@ -300,7 +285,7 @@ python-slugify==4.0.1
 python-speech-features==0.6
 python-xlib==0.29
 pythonWebHDFS==0.2.3
-pythran==0.11.0 
+pythran 
 pyvista==0.24.3
 pyvo==1.0
 qgrid==1.3.1
@@ -352,13 +337,13 @@ threadpool==1.3.2
 threadpoolctl==2.1.0
 tomlkit==0.7.0
 toolz==0.10.0
-torch==1.11.0
+torch
 transformers==4.12.5
 tweepy==3.10.0
 typed-ast==1.4.3
 typing==3.7.4.3
 typing-extensions==3.7.4.3
-ujson==5.1.0
+ujson
 update-checker==0.16
 uritemplate==3.0.1
 urllib3==1.26.7
@@ -369,7 +354,7 @@ vtk==9.0.1
 websocket-client==0.59.0
 wheel==0.36.2
 widgetsnbextension==3.5.1
-xgboost==1.5.2
+xgboost
 yapf==0.29.0
 yarl==1.4.2
 zict==1.0.0

vendor/ngc-tensorflow/requirements.22.03.txt

@@ -156,39 +156,23 @@ Jinja2==2.11.3
 json-tricks==3.14.0
 jsonpickle==1.5.2
 jsonschema==4.0.1
-jupyter==1.0.0
-jupyter-bokeh==2.0.4
-jupyter-client==6.1.12
-jupyter-console==6.2.0
+jupyter-console==6.1.0
 jupyter-contrib-core==0.3.3
 jupyter-contrib-nbextensions==0.5.1
-jupyter-core==4.9.2
 jupyter-dash==0.4.0
 jupyter-highlight-selected-word==0.2.0
 jupyter-js-widgets-nbextension==0.0.2.dev0
-jupyter-kite==1.2.0
 jupyter-latex-envs==1.4.6
 jupyter-lsp==1.2.0
 jupyter-nbextensions-configurator==0.4.1
-jupyter-packaging==0.9.2
-jupyter-server==1.6.4
-jupyter-server-mathjax==0.2.2
-jupyter-server-proxy==1.6.0
-jupyter-telemetry==0.1.0
-jupyterhub==2.2.2
-jupyterhub-jwtauthenticator==0.1
-jupyterhub-kubespawner==0.16.1
-jupyterhubutils==0.25.0
-jupyterlab==3.3.2
-jupyterlab-code-formatter==1.4.10
-jupyterlab-git==0.30.1
+jupyter-server-proxy==1.2.0
+jupyter==1.0.0
+jupyter_bokeh==2.0.4
+jupyterlab-git
 jupyterlab-github==2.0.0
-jupyterlab-hdf==0.5.1
-jupyterlab-launcher==0.13.1
 jupyterlab-lsp==3.6.0
-jupyterlab-pygments==0.1.2
-jupyterlab-server==2.10.3
-jupyterlabutils==0.9.2
+jupyterlab_hdf==0.3.0
+jupyterlabutils
 jupyterthemes==0.20.0
 jupytext==1.13.7
 keras==2.8.0