Merge pull request #380 from Xaenalt/main

Add Habana 1.13 notebook

Author: harshad16

Makefile

@@ -114,6 +114,11 @@ habana-jupyter-1.10.0-ubi8-python-3.8: jupyter-datascience-ubi8-python-3.8
 habana-jupyter-1.11.0-ubi8-python-3.8: jupyter-datascience-ubi8-python-3.8
 	$(call image,$@,habana/1.11.0/ubi8-python-3.8,$<)
 
+# Build and push habana-jupyter-1.13.0-ubi8-python-3.8 image to the registry
+.PHONY: habana-jupyter-1.13.0-ubi8-python-3.8
+habana-jupyter-1.13.0-ubi8-python-3.8: jupyter-datascience-ubi8-python-3.8
+	$(call image,$@,habana/1.13.0/ubi8-python-3.8,$<)
+
 # Build and push runtime-minimal-ubi8-python-3.8 image to the registry
 .PHONY: runtime-minimal-ubi8-python-3.8
 runtime-minimal-ubi8-python-3.8: base-ubi8-python-3.8

habana/1.13.0/ubi8-python-3.8/Dockerfile

@@ -0,0 +1,167 @@
+# Copyright (c) 2022 Habana Labs, Ltd.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# HabanaLabs Dockerfile base installer layer for RedHat 8.6
+# Reference: https://github.com/HabanaAI/Setup_and_Install/blob/1.13.0/dockerfiles/base/Dockerfile.rhel8.6 
+ARG BASE_IMAGE
+FROM ${BASE_IMAGE}
+
+ARG ARTIFACTORY_URL="vault.habana.ai"
+ARG VERSION="1.13.0"
+ARG REVISION="463"
+ARG PT_VERSION="2.1.0"
+
+USER root
+
+RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
+    dnf clean all && rm -rf /var/cache/yum
+
+RUN echo "[appstream]" > /etc/yum.repos.d/CentOS-Linux-AppStream.repo && \
+    echo "name=CentOS Linux 8 - AppStream" >> /etc/yum.repos.d/CentOS-Linux-AppStream.repo && \
+    echo "mirrorlist=http://mirrorlist.centos.org/?release=\$releasever-stream&arch=\$basearch&repo=AppStream&infra=\$infra" >> /etc/yum.repos.d/CentOS-Linux-AppStream.repo && \
+    echo "gpgcheck=0" >> /etc/yum.repos.d/CentOS-Linux-AppStream.repo
+
+
+RUN echo "[BaseOS]" > /etc/yum.repos.d/CentOS-Linux-BaseOS.repo && \
+    echo "name=CentOS Linux 8 - BaseOS" >> /etc/yum.repos.d/CentOS-Linux-BaseOS.repo && \
+    echo "mirrorlist=http://mirrorlist.centos.org/?release=\$releasever-stream&arch=\$basearch&repo=BaseOS&infra=\$infra" >> /etc/yum.repos.d/CentOS-Linux-BaseOS.repo && \
+    echo "gpgcheck=0" >> /etc/yum.repos.d/CentOS-Linux-BaseOS.repo
+
+RUN dnf install -y \
+    clang \
+    cmake3 \
+    cpp \
+    gcc \
+    gcc-c++ \
+    glibc \
+    glibc-headers \
+    glibc-devel \
+    jemalloc \
+    libarchive \
+    libksba \
+    unzip \
+    llvm \
+    lsof \
+    python38-devel \
+    openssh-clients \
+    libjpeg-devel \
+    openssh-server \
+    redhat-lsb-core \
+    wget \
+    git \
+    mesa-libGL \
+    python3-dnf-plugin-versionlock && \
+    # update pkgs (except OS version) for resolving potentials CVEs
+    dnf versionlock add redhat-release* && \
+    dnf update -y && \
+    dnf clean all && rm -rf /var/cache/yum
+
+ENV LD_LIBRARY_PATH=/usr/lib/habanalabs:$LD_LIBRARY_PATH
+ENV RDMAV_FORK_SAFE=1
+
+RUN echo "[habanalabs]" > /etc/yum.repos.d/habanalabs.repo && \
+    echo "name=Habana RH8 Linux repo" >> /etc/yum.repos.d/habanalabs.repo && \
+    echo "baseurl=https://${ARTIFACTORY_URL}/artifactory/rhel/8/8.6" >> /etc/yum.repos.d/habanalabs.repo && \
+    echo "gpgkey=https://${ARTIFACTORY_URL}/artifactory/rhel/8/8.6/repodata/repomd.xml.key" >> /etc/yum.repos.d/habanalabs.repo
+
+RUN echo "[powertools]" > /etc/yum.repos.d/powertools.repo && \
+    echo "name=powertools" >> /etc/yum.repos.d/powertools.repo && \
+    echo "baseurl=http://mirror.centos.org/centos/8-stream/PowerTools/x86_64/os/"  >> /etc/yum.repos.d/powertools.repo && \
+    echo "gpgcheck=0" >> /etc/yum.repos.d/powertools.repo
+
+RUN dnf install -y habanalabs-rdma-core-"$VERSION"-"$REVISION".el8 \
+        habanalabs-thunk-"$VERSION"-"$REVISION".el8 \
+        habanalabs-firmware-tools-"$VERSION"-"$REVISION".el8 \
+        habanalabs-graph-"$VERSION"-"$REVISION".el8 && \
+    rm -f /etc/yum.repos.d/habanalabs.repo && rm -rf /tmp/* && \
+    dnf clean all && rm -rf /var/cache/yum
+
+RUN rpm -V habanalabs-rdma-core && rpm -V habanalabs-thunk && rpm -V habanalabs-firmware-tools && rpm -V habanalabs-graph
+
+# There is no need to store pip installation files inside docker image
+ENV PIP_NO_CACHE_DIR=on
+ENV PIP_DISABLE_PIP_VERSION_CHECK=1
+ENV RDMA_CORE_ROOT=/opt/habanalabs/rdma-core/src
+ENV RDMA_CORE_LIB=${RDMA_CORE_ROOT}/build/lib
+
+# Install python packages 
+# RUN python3.8 -m pip install hpu_media_loader=="${VERSION}"."${REVISION}"
+# Install Python packages and Jupyterlab extensions from Pipfile.lock
+COPY Pipfile.lock ./
+
+RUN echo "Installing softwares and packages" && micropipenv install && rm -f ./Pipfile.lock
+
+RUN echo "export LANG=en_US.UTF-8" >> /root/.bashrc
+RUN export LANG=en_US.UTF-8
+ENV GC_KERNEL_PATH=/usr/lib/habanalabs/libtpc_kernels.so
+ENV HABANA_LOGS=/var/log/habana_logs/
+ENV HABANA_SCAL_BIN_PATH=/opt/habanalabs/engines_fw
+ENV HABANA_PLUGINS_LIB_PATH=/opt/habanalabs/habana_plugins
+
+## Install habana tensorflow
+# Reference: https://github.com/HabanaAI/Setup_and_Install/blob/1.13.0/dockerfiles/tensorflow/Dockerfile.rhel8.6
+# For AML/CentOS/RHEL OS'es TFIO_DATAPATH have to be specified to import tensorflow_io lib correctly
+ENV TFIO_DATAPATH=/opt/app-root/src/python3.8/site-packages/
+
+# For AML/CentOS/RHEL ca-cert file is expected exactly under /etc/ssl/certs/ca-certificates.crt
+# otherwise curl will fail during access to S3 AWS storage
+RUN ln -s /etc/ssl/certs/ca-bundle.crt /etc/ssl/certs/ca-certificates.crt
+
+## Install habana pytorch
+# Reference: https://github.com/HabanaAI/Setup_and_Install/blob/1.13.0/dockerfiles/pytorch/Dockerfile.rhel8.6
+ENV LANG=en_US.UTF-8
+ENV PYTHONPATH=/root:/usr/lib/habanalabs/
+
+RUN dnf install -y \
+    curl \
+    cairo-devel \
+    numactl-devel \
+    iproute \
+    which \
+    zlib-devel \
+    lapack-devel \
+    openblas-devel \
+    numactl \
+    gperftools-devel && \
+    dnf clean all && rm -rf /var/cache/yum
+
+RUN wget --no-verbose https://"${ARTIFACTORY_URL}"/artifactory/gaudi-pt-modules/"${VERSION}"/"${REVISION}"\
+/pytorch/rhel86/pytorch_modules-v"${PT_VERSION}"_"${VERSION}"_"${REVISION}".tgz && \
+    mkdir /root/habanalabs /root/habanalabs/pytorch_temp && \
+    tar -xf pytorch_modules-v"${PT_VERSION}"_"${VERSION}"_"${REVISION}".tgz -C /root/habanalabs/pytorch_temp/. && \
+    pip3 install /root/habanalabs/pytorch_temp/*.whl && \
+    pip3 install $(grep "lightning" /root/habanalabs/pytorch_temp/requirements-pytorch.txt) && \
+    pip3 install tensorboard~=2.12.2 protobuf==3.20.3 && \
+    pip3 uninstall -y pillow && \
+    pip3 uninstall -y pillow-simd && \
+    pip3 install pillow-simd==7.0.0.post3 && \
+    rm -rf /root/habanalabs/pytorch_temp/ && \
+    rm -rf pytorch_modules-v"${PT_VERSION}"_"${VERSION}"_"${REVISION}".tgz &&\
+    echo "source /etc/profile.d/habanalabs.sh" >> ~/.bashrc
+
+ENV LD_PRELOAD=/lib64/libtcmalloc.so.4
+ENV TCMALLOC_LARGE_ALLOC_REPORT_THRESHOLD=7516192768
+
+RUN dnf clean all && rm -rf /var/cache/dnf && rm -rf /tmp/*
+
+## Label the image with details required by ODH
+LABEL name="odh-notebook-habana-jupyter-1.3.0-ubi8-python-3.8" \
+    summary="Jupyter HabanaAI 1.13.0 notebook image for ODH notebooks" \
+    description="Jupyter HabanaAI 1.13.0 notebook image with base Python 3.8 builder image based on ubi8 for ODH notebooks" \
+    io.k8s.display-name="Jupyter HabanaAI 1.13.0 notebook image for ODH notebooks" \
+    io.k8s.description="Jupyter HabanaAI 1.13.0 notebook image with base Python 3.8 builder image based on ubi8 for ODH notebooks" \
+    authoritative-source-url="https://github.com/opendatahub-io/notebooks" \
+    io.openshift.build.commit.ref="main" \
+    io.openshift.build.source-location="https://github.com/opendatahub-io/notebooks/tree/main/habana/1.13.0/ubi8-python-3.8" \
+    io.openshift.build.image="quay.io/opendatahub/workbench-images:habana-jupyter-1.13.0-ubi8-python-3.8"
+
+# Replace Notebook's launcher, "(ipykernel)" with Python's version 3.x
+RUN sed -i -e "s/Python.*/$(python --version| cut -d '.' -f-2)\",/" /opt/app-root/share/jupyter/kernels/python3/kernel.json && \
+    # Fix permissions to support pip in Openshift environments \
+    chmod -R g+w /opt/app-root/lib/python3.8/site-packages && \
+    fix-permissions /opt/app-root -P
+
+USER 1001
+
+WORKDIR /opt/app-root/src

habana/1.13.0/ubi8-python-3.8/Pipfile

@@ -0,0 +1,61 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[dev-packages]
+
+[packages]
+# Habana requirements
+habana_media_loader = "==1.13.0.463"
+
+# TensorFlow and useful extensions
+habana-tensorflow = "==1.13.0.463"
+tensorflow-cpu = "==2.12.1"
+tensorflow-io = "==0.32.0"
+tensorboard = "~=2.12.2"
+tf2onnx = "~= 1.13.0"
+
+# Pytorch and useful extension
+habana-pyhlml = "==1.13.0.463"
+
+# Below pkgs are required for pytorch-fork
+cffi = "==1.15.1"
+typing-extensions = ">=4.5.0"
+numpy = "==1.23.5"
+ninja = ">=1.11.1"
+pyyaml = "==6.0"
+pybind11 = "==2.10.4"
+mkl = "==2023.1"
+mkl-include = "==2023.1"
+
+# Parent image requirements to maintain cohesion
+elyra-pipeline-editor-extension = "~=3.15.0"
+elyra-python-editor-extension = "~=3.15.0"
+elyra-code-snippet-extension = "~=3.15.0"
+kfp-tekton = "<1.6.0"
+boto3 = "~=1.26.69"
+jupyter-bokeh = "~=3.0.5"
+kafka-python = "~=2.0.2"
+matplotlib = "~=3.6.3"
+pandas = "~=1.5.3"
+plotly = "~=5.13.0"
+scikit-learn = "~=1.2.1"
+scipy = "~=1.10.0"
+jupyterlab-lsp = "~=3.10.2"
+jupyterlab-widgets = "~=3.0.5"
+jupyter-resource-usage = "~=0.6.0"
+
+# Parent image requirements to maintain cohesion
+jupyterlab = "~=3.5.3"
+jupyter-server = "~=2.1.0"
+jupyter-server-proxy = "~=3.2.2"
+jupyter-server-terminals = "~=0.4.4"
+jupyterlab-git = "~=0.41.0"
+nbdime = "~=3.1.1"
+nbgitpuller = "~=1.1.1"
+# ---
+wheel = "~=0.38.4"
+
+[requires]
+python_version = "3.8"