Skip to content

Commit bb8c8ca

Browse files
harshad16harshad16
authored
Merge pull request #428 from sharvil10/intel_pytorch_2024.0
Intel® PyTorch notebook container
2 parents ac54cf2 + 3b01196 commit bb8c8ca

29 files changed

+20250
-5
lines changed

‎Makefile

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,20 @@ runtime-cuda-tensorflow-ubi9-python-3.9: cuda-ubi9-python-3.9
187187
codeserver-ubi9-python-3.9: base-ubi9-python-3.9
188188
$(call image,$@,codeserver/ubi9-python-3.9,$<)
189189

190+
# Build and push base-anaconda-python-3.9-intel-gpu image to the registry
191+
.PHONY: intel-base-gpu-ubi9-python-3.9
192+
intel-base-gpu-ubi9-python-3.9: base-ubi9-python-3.9
193+
$(call image,$@,intel/base/gpu/ubi9-python-3.9,$<)
194+
195+
# Build and push intel-runtime-pytorch-ubi9-python-3.9 image to the registry
196+
.PHONY: intel-runtime-pytorch-ubi9-python-3.9
197+
intel-runtime-pytorch-ubi9-python-3.9: intel-base-gpu-ubi9-python-3.9
198+
$(call image,$@,intel/runtimes/pytorch/ubi9-python-3.9,$<)
199+
200+
# Build and push jupyter-intel-pytorch-ubi9-python-3.9 image to the registry
201+
.PHONY: jupyter-intel-pytorch-ubi9-python-3.9
202+
jupyter-intel-pytorch-ubi9-python-3.9: intel-runtime-pytorch-ubi9-python-3.9
203+
$(call image,$@,jupyter/intel/pytorch/ubi9-python-3.9,$<)
190204

191205
####################################### Buildchain for Python 3.9 using C9S #######################################
192206

@@ -308,24 +322,24 @@ undeploy-c9s-%-c9s-python-3.9: bin/kubectl
308322
# ARG 1: UBI flavor
309323
# ARG 1: Python kernel
310324
define test_with_papermill
325+
$(eval PREFIX_NAME := $(subst /,-,$(1)_$(2))) \
311326
$(KUBECTL_BIN) exec $(FULL_NOTEBOOK_NAME) -- /bin/sh -c "python3 -m pip install papermill" ; \
312-
$(KUBECTL_BIN) exec $(FULL_NOTEBOOK_NAME) -- /bin/sh -c "wget ${NOTEBOOK_REPO_BRANCH_BASE}/jupyter/$(1)/$(2)-$(3)/test/test_notebook.ipynb -O test_notebook.ipynb && python3 -m papermill test_notebook.ipynb $(1)_$(2)_output.ipynb --kernel python3 --stderr-file $(1)_$(2)_error.txt" ; \
327+
$(KUBECTL_BIN) exec $(FULL_NOTEBOOK_NAME) -- /bin/sh -c "wget ${NOTEBOOK_REPO_BRANCH_BASE}/jupyter/$(1)/$(2)-$(3)/test/test_notebook.ipynb -O test_notebook.ipynb && python3 -m papermill test_notebook.ipynb $(PREFIX_NAME)_output.ipynb --kernel python3 --stderr-file $(PREFIX_NAME)_error.txt" ; \
313328
if [ $$? -ne 0 ]; then \
314-
echo "ERROR: The $(1) $(2) notebook encountered a failure. To investigate the issue, you can review the logs located in the ocp-ci cluster on 'artifacts/notebooks-e2e-tests/jupyter-$(1)-$(2)-$(3)-test-e2e' directory or run 'cat $(1)_$(2)_error.txt' within your container. The make process has been aborted." ; \
329+
echo "ERROR: The $(1) $(2) notebook encountered a failure. To investigate the issue, you can review the logs located in the ocp-ci cluster on 'artifacts/notebooks-e2e-tests/jupyter-$(1)-$(2)-$(3)-test-e2e' directory or run 'cat $(PREFIX_NAME)_error.txt' within your container. The make process has been aborted." ; \
315330
exit 1 ; \
316331
fi ; \
317-
$(KUBECTL_BIN) exec $(FULL_NOTEBOOK_NAME) -- /bin/sh -c "cat $(1)_$(2)_error.txt | grep --quiet FAILED" ; \
332+
$(KUBECTL_BIN) exec $(FULL_NOTEBOOK_NAME) -- /bin/sh -c "cat $(PREFIX_NAME)_error.txt | grep --quiet FAILED" ; \
318333
if [ $$? -eq 0 ]; then \
319334
echo "ERROR: The $(1) $(2) notebook encountered a failure. The make process has been aborted." ; \
320-
$(KUBECTL_BIN) exec $(FULL_NOTEBOOK_NAME) -- /bin/sh -c "cat $(1)_$(2)_error.txt" ; \
335+
$(KUBECTL_BIN) exec $(FULL_NOTEBOOK_NAME) -- /bin/sh -c "cat $(PREFIX_NAME)_error.txt" ; \
321336
exit 1 ; \
322337
fi
323338
endef
324339

325340
# Verify the notebook's readiness by pinging the /api endpoint and executing the corresponding test_notebook.ipynb file in accordance with the build chain logic.
326341
.PHONY: test
327342
test-%: bin/kubectl
328-
329343
# Verify the notebook's readiness by pinging the /api endpoint
330344
$(eval NOTEBOOK_NAME := $(subst .,-,$(subst cuda-,,$*)))
331345
$(info # Running tests for $(NOTEBOOK_NAME) notebook...)
@@ -336,6 +350,8 @@ test-%: bin/kubectl
336350
# Tests notebook's functionalities
337351
if echo "$(FULL_NOTEBOOK_NAME)" | grep -q "minimal-ubi9"; then \
338352
$(call test_with_papermill,minimal,ubi9,python-3.9) \
353+
elif echo "$(FULL_NOTEBOOK_NAME)" | grep -q "intel-pytorch-ubi9"; then \
354+
$(call test_with_papermill,intel/pytorch,ubi9,python-3.9) \
339355
elif echo "$(FULL_NOTEBOOK_NAME)" | grep -q "datascience-ubi9"; then \
340356
$(MAKE) validate-ubi9-datascience -e FULL_NOTEBOOK_NAME=$(FULL_NOTEBOOK_NAME); \
341357
elif echo "$(FULL_NOTEBOOK_NAME)" | grep -q "pytorch-ubi9"; then \

‎intel/base/gpu/ubi9-python-3.9/Dockerfile

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
ARG BASE_IMAGE
2+
FROM ${BASE_IMAGE}
3+
4+
LABEL name="odh-notebook-intel-base-gpu-ubi9-python-3.9" \
5+
summary="Intel® XPU base image for ODH notebooks based on ubi9" \
6+
description="Base Intel® XPU base image builder image based on ubi9 for ODH notebooks" \
7+
io.k8s.display-name="Base Intel® XPU base image for ODH notebooks based on ubi9" \
8+
io.k8s.description="Base Intel® XPU base image builder image based on ubi9 for ODH notebooks" \
9+
authoritative-source-url="https://github.com/opendatahub-io/notebooks" \
10+
io.openshift.build.commit.ref="main" \
11+
io.openshift.build.source-location="https://github.com/opendatahub-io/notebooks/tree/main/intel/base/gpu/ubi9-python-3.9" \
12+
io.openshift.build.image="quay.io/opendatahub/workbench-images:intel-base-gpu-ubi9-python-3.9"
13+
14+
USER 0
15+
WORKDIR /opt/app-root/src
16+
17+
RUN . /etc/os-release && \
18+
dnf install -y 'dnf-command(config-manager)' && \
19+
dnf config-manager --add-repo \
20+
https://repositories.intel.com/gpu/rhel/${VERSION_ID}/lts/2350/unified/intel-gpu-${VERSION_ID}.repo && \
21+
dnf install -y \
22+
intel-opencl \
23+
level-zero intel-level-zero-gpu level-zero-devel \
24+
intel-metrics-discovery \
25+
intel-metrics-library && \
26+
dnf clean all -y && \
27+
rm -rf /var/cache/dnf/*
28+
29+
RUN dnf update -y && \
30+
rpm -ivh https://mirror.stream.centos.org/9-stream/BaseOS/x86_64/os/Packages/numactl-libs-2.0.16-3.el9.x86_64.rpm && \
31+
rpm -ivh https://mirror.stream.centos.org/9-stream/BaseOS/x86_64/os/Packages/numactl-2.0.16-3.el9.x86_64.rpm && \
32+
dnf install -y \
33+
git \
34+
rsync \
35+
unzip \
36+
wget && \
37+
dnf clean all && \
38+
rm -rf /var/cache/dnf/*
39+
40+
41+
USER 1001

‎intel/runtimes/pytorch/ubi9-python-3.9/Dockerfile

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
ARG BASE_IMAGE
2+
FROM ${BASE_IMAGE}
3+
4+
LABEL name="odh-notebook-intel-runtime-pytorch-ubi9-python-3.9" \
5+
summary="Runtime Intel® optimized pytorch notebook image for ODH notebooks" \
6+
description="Runtime Intel® optimized pytorch notebook image with base Python 3.9 builder image based on UBI9 for ODH notebooks" \
7+
io.k8s.display-name="Runtime Intel® optimized pytorch notebook image for ODH notebooks" \
8+
io.k8s.description="Runtime Intel® optimized pytorch notebook image with base Python 3.9 builder image based on UBI9 for ODH notebooks" \
9+
authoritative-source-url="https://github.com/opendatahub-io/notebooks" \
10+
io.openshift.build.commit.ref="main" \
11+
io.openshift.build.source-location="https://github.com/opendatahub-io/notebooks/tree/main/intel/runtimes/pytorch/ubi9-python-3.9" \
12+
io.openshift.build.image="quay.io/opendatahub/workbench-images:intel-runtime-pytorch-ubi9-python-3.9"
13+
14+
USER root
15+
16+
ARG ONEAPI_VERSION=2024.0
17+
COPY oneAPI.repo /etc/yum.repos.d/oneAPI.repo
18+
19+
RUN dnf update -y && \
20+
dnf install -y intel-oneapi-dpcpp-cpp-${ONEAPI_VERSION} \
21+
intel-oneapi-mkl-${ONEAPI_VERSION} && \
22+
dnf clean all -y && \
23+
rm -rf /var/cache/dnf/* && \
24+
rm /etc/yum.repos.d/oneAPI.repo
25+
26+
USER 1001
27+
28+
WORKDIR /opt/app-root/bin
29+
30+
# Install Python packages from Pipfile.lock
31+
COPY Pipfile.lock.gpu Pipfile.lock
32+
33+
# Copy Elyra dependencies for air-gapped enviroment
34+
COPY utils utils
35+
36+
RUN echo "Installing softwares and packages" && \
37+
micropipenv install && \
38+
rm -f ./Pipfile.lock && \
39+
# Fix permissions to support pip in Openshift environments \
40+
chmod -R g+w /opt/app-root/lib/python3.9/site-packages && \
41+
fix-permissions /opt/app-root -P
42+
43+
ENV CPU_ENV=/opt/app-root-cpu
44+
COPY --chown=1001:0 Pipfile.lock.cpu ${CPU_ENV}/Pipfile.lock
45+
46+
#CPU env
47+
RUN cd ${CPU_ENV} && \
48+
python3.9 -m venv ${CPU_ENV} && \
49+
chown -R 1001:0 ${CPU_ENV} && \
50+
fix-permissions ${CPU_ENV} -P && \
51+
source ${CPU_ENV}/bin/activate && \
52+
pip install -U "micropipenv[toml]" && \
53+
micropipenv install && \
54+
rm -f ./Pipfile.lock && \
55+
# Fix permissions to support pip in Openshift environments \
56+
chmod -R g+w /opt/app-root/lib/python3.9/site-packages && \
57+
fix-permissions /opt/app-root -P
58+
59+
WORKDIR /opt/app-root/src

‎intel/runtimes/pytorch/ubi9-python-3.9/Pipfile.cpu

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
[[source]]
2+
url = "https://pypi.org/simple"
3+
verify_ssl = true
4+
name = "pypi"
5+
6+
[[source]]
7+
url = "https://download.pytorch.org/whl/cpu"
8+
verify_ssl = true
9+
name = "torch-cpu"
10+
11+
[[source]]
12+
url = "https://pytorch-extension.intel.com/release-whl/stable/cpu/us/"
13+
verify_ssl = true
14+
name = "ipex-cpu"
15+
16+
[packages]
17+
# PyTorch packages
18+
torch = {version="==2.1.2", index="torch-cpu"}
19+
intel-extension-for-pytorch = {version="==2.1.100", index="ipex-cpu"}
20+
torchvision = {version="0.16.2", index="torch-cpu"}
21+
torchaudio = {version="2.1.2", index="torch-cpu"}
22+
neural-compressor = {version="2.4.1"}
23+
tensorboard = {version="*"}
24+
25+
# LLM Packages
26+
datasets = {version="*"}
27+
transformers = {version="*"}
28+
evaluate = {version="*"}
29+
30+
# Datascience and useful extensions
31+
cloud-data-connector = {version="*"}
32+
kafka-python = "~=2.0.2"
33+
matplotlib = "~=3.6.3"
34+
pandas = "~=1.5.3"
35+
plotly = "~=5.16.1"
36+
scikit-learn = "*"
37+
scipy = "~=1.11.2"
38+
skl2onnx = "~=1.15.0"
39+
codeflare-sdk = "~=0.13.0"
40+
41+
# DB connectors
42+
pymongo = "~=4.5.0"
43+
psycopg = "~=3.1.10"
44+
pyodbc = "~=4.0.39"
45+
mysql-connector-python = "~=8.0.33"
46+
47+
# This is a comprehensive list of python dependencies that Elyra requires to execute Jupyter notebooks.
48+
ipykernel = "==6.13.0"
49+
ipython = "==8.10.0"
50+
ipython-genutils = "==0.2.0"
51+
jinja2 = "==3.0.3"
52+
jupyter-client = "==7.3.1"
53+
jupyter-core = "==4.11.2"
54+
MarkupSafe = "==2.1.1"
55+
minio = "==7.1.15"
56+
nbclient = "==0.6.3"
57+
nbconvert = "==6.5.1"
58+
nbformat = "==5.4.0"
59+
papermill = "==2.3.4"
60+
pyzmq = "==24.0.1"
61+
prompt-toolkit = "==3.0.30"
62+
requests = "==2.31.0"
63+
tornado = "==6.3.3"
64+
traitlets = "==5.1.1"
65+
urllib3 = "==1.26.9"
66+
67+
[requires]
68+
python_version = "3.9"

‎intel/runtimes/pytorch/ubi9-python-3.9/Pipfile.gpu

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
[[source]]
2+
url = "https://pypi.org/simple"
3+
verify_ssl = true
4+
name = "pypi"
5+
6+
[[source]]
7+
url = "https://pytorch-extension.intel.com/release-whl/stable/xpu/us/"
8+
verify_ssl = true
9+
name = "ipex-xpu"
10+
11+
[packages]
12+
# PyTorch packages
13+
torch = {version="==2.1.0a0", index="ipex-xpu"}
14+
intel_extension_for_pytorch = {version="==2.1.10+xpu", index="ipex-xpu"}
15+
torchvision = {version="0.16.0a0", index="ipex-xpu"}
16+
torchaudio = {version="2.1.0a0", index="ipex-xpu"}
17+
neural-compressor = {version="2.4.1"}
18+
tensorboard = {version="*"}
19+
20+
# LLM Packages
21+
datasets = {version="*"}
22+
transformers = {version="*"}
23+
evaluate = {version="*"}
24+
25+
# Datascience and useful extensions
26+
cloud-data-connector = {version="*"}
27+
kafka-python = "~=2.0.2"
28+
matplotlib = "~=3.6.3"
29+
pandas = "~=1.5.3"
30+
plotly = "~=5.16.1"
31+
scikit-learn = "*"
32+
scipy = "~=1.11.2"
33+
skl2onnx = "~=1.15.0"
34+
codeflare-sdk = "~=0.13.0"
35+
36+
# DB connectors
37+
pymongo = "~=4.5.0"
38+
psycopg = "~=3.1.10"
39+
pyodbc = "~=4.0.39"
40+
mysql-connector-python = "~=8.0.33"
41+
42+
# This is a comprehensive list of python dependencies that Elyra requires to execute Jupyter notebooks.
43+
ipykernel = "==6.13.0"
44+
ipython = "==8.10.0"
45+
ipython-genutils = "==0.2.0"
46+
jinja2 = "==3.0.3"
47+
jupyter-client = "==7.3.1"
48+
jupyter-core = "==4.11.2"
49+
MarkupSafe = "==2.1.1"
50+
minio = "==7.1.15"
51+
nbclient = "==0.6.3"
52+
nbconvert = "==6.5.1"
53+
nbformat = "==5.4.0"
54+
papermill = "==2.3.4"
55+
pyzmq = "==24.0.1"
56+
prompt-toolkit = "==3.0.30"
57+
requests = "==2.31.0"
58+
tornado = "==6.3.3"
59+
traitlets = "==5.1.1"
60+
urllib3 = "==1.26.9"
61+
62+
[requires]
63+
python_version = "3.9"

0 commit comments

Comments
 (0)