Merge pull request #352 from harshad16/rsync-2024a

[2024a] release-2024a from opendatahub-io:2024a

Author: harshad16

.github/workflows/build-notebooks-TEMPLATE.yaml

@@ -23,7 +23,7 @@ jobs:
       # GitHub image registry used for storing $(CONTAINER_ENGINE)'s cache
       CACHE: "ghcr.io/${{ github.repository }}/workbench-images/build-cache"
       # Targets (and their folder) that should be scanned using FS instead of IMAGE scan due to resource constraints
-      TRIVY_SCAN_FS_JSON: '{"rocm-jupyter-pytorch-ubi9-python-3.9": "jupyter/rocm/pytorch/ubi9-python-3.9"}'
+      TRIVY_SCAN_FS_JSON: '{}'
 
     steps:
 
@@ -123,24 +123,23 @@ jobs:
         env:
           IMAGE_REGISTRY: "ghcr.io/${{ github.repository }}/workbench-images"
           CONTAINER_BUILD_CACHE_ARGS: "--cache-from ${{ env.CACHE }} --cache-to ${{ env.CACHE }}"
+          # dependent images were already built and pushed, so just let podman pull it
+          BUILD_DEPENDENT_IMAGES: "no"
 
       # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request
       - name: "pull_request: make ${{ inputs.target }}"
         run: |
-          # start a black hole container registry as make target always does a push
-          mkdir -p $HOME/.config/containers/registries.conf.d/
-          cp ci/cached-builds/insecure_localhost_registry.conf $HOME/.config/containers/registries.conf.d/insecure_localhost_registry.conf
-          go run ci/cached-builds/dev_null_container_registry.go &
-          # build and push the image
           make ${{ inputs.target }}
         if: "${{ fromJson(inputs.github).event_name == 'pull_request' }}"
         env:
           IMAGE_TAG: "${{ github.sha }}"
           IMAGE_REGISTRY: "localhost:5000/workbench-images"
           CONTAINER_BUILD_CACHE_ARGS: "--cache-from ${{ env.CACHE }}"
+          # We don't have access to image registry, so disable pushing
+          PUSH_IMAGES: "no"
 
       - name: "Show podman images information"
-        run: podman images
+        run: podman images --digests
 
       - name: "pull_request|schedule: resolve target if Trivy scan should run"
         id: resolve-target

Makefile

@@ -2,6 +2,10 @@ IMAGE_REGISTRY   ?= quay.io/opendatahub/workbench-images
 RELEASE	 		 ?= 2024a
 # additional user-specified caching parameters for $(CONTAINER_ENGINE) build
 CONTAINER_BUILD_CACHE_ARGS ?= --no-cache
+# whether to build all dependent images or just the one specified
+BUILD_DEPENDENT_IMAGES ?= yes
+# whether to push the images to a registry as they are built
+PUSH_IMAGES ?= yes
 
 # OS dependant: Generate date, select appropriate cmd to locate container engine
 ifeq ($(OS), Windows_NT)
@@ -58,10 +62,19 @@ endef
 #   ARG 1: Image tag name.
 #   ARG 2: Path of image context we want to build.
 #   ARG 3: Base image tag name (optional).
+#
+# BUILD_DEPENDENT_IMAGES: only build images that were explicitly given as a goal on command line
+# PUSH_IMAGES: allows skipping podman push
 define image
 	$(info #*# Image build directory: <$(2)> #(MACHINE-PARSED LINE)#*#...)
-	$(call build_image,$(1),$(2),$(3))
-	$(call push_image,$(1))
+
+	$(if $(or $(BUILD_DEPENDENT_IMAGES:no=), $(filter $@,$(MAKECMDGOALS))),
+		$(call build_image,$(1),$(2),$(3))
+
+		$(if $(PUSH_IMAGES:no=),
+			$(call push_image,$(1))
+		)
+	)
 endef
 
 ####################################### Buildchain for Python 3.8 using ubi8 #######################################

README.md

@@ -37,7 +37,9 @@ To build a workbench image, you can execute the following command:
 make ${WORKBENCH_NAME} -e  IMAGE_REGISTRY=quay.io/${YOUR_USER}/workbench-images  -e  RELEASE=2023x
 ```
 
-Using  `IMAGE_REGISTRY` and `RELEASE` variables  you can overwrite the default values and use a different registry or release tag
+Using  `IMAGE_REGISTRY` and `RELEASE` variables you can overwrite the default values and use a different registry or release tag
+
+Using `CONTAINER_BUILD_CACHE_ARGS` (default: `--no-cache`), `BUILD_DEPENDENT_IMAGES`, and `PUSH_IMAGES` variables you can further customize the build process.
 
 ### Local Execution
 

ci/cached-builds/dev_null_container_registry.go

@@ -20,7 +20,10 @@ COPY setup-elyra.sh ./utils/
 
 RUN echo "Installing softwares and packages" && \
     micropipenv install && \
-    rm -f ./Pipfile.lock
+    rm -f ./Pipfile.lock && \
+    # Fix permissions to support pip in Openshift environments \
+    chmod -R g+w /opt/app-root/lib/python3.9/site-packages && \
+    fix-permissions /opt/app-root -P
 
 COPY utils ./utils/
 

ci/cached-builds/insecure_localhost_registry.conf

@@ -18,14 +18,12 @@ COPY utils utils/
 COPY Pipfile.lock start-notebook.sh ./
 
 # Install Python dependencies from Pipfile.lock file
-RUN echo "Installing softwares and packages" && micropipenv install && rm -f ./Pipfile.lock
-
-# Disable announcement plugin of jupyterlab
-RUN jupyter labextension disable "@jupyterlab/apputils-extension:announcements"
-
-# Fix permissions to support pip in Openshift environments
-RUN chmod -R g+w /opt/app-root/lib/python3.8/site-packages && \
-      fix-permissions /opt/app-root -P
+RUN echo "Installing softwares and packages" && micropipenv install && rm -f ./Pipfile.lock && \
+    # Disable announcement plugin of jupyterlab \
+    jupyter labextension disable "@jupyterlab/apputils-extension:announcements" && \
+    # Fix permissions to support pip in Openshift environments \
+    chmod -R g+w /opt/app-root/lib/python3.8/site-packages && \
+    fix-permissions /opt/app-root -P
 
 WORKDIR /opt/app-root/src
 

jupyter/datascience/ubi9-python-3.9/Dockerfile

@@ -18,14 +18,12 @@ COPY utils utils/
 COPY Pipfile.lock start-notebook.sh ./
 
 # Install Python dependencies from Pipfile.lock file
-RUN echo "Installing softwares and packages" && micropipenv install && rm -f ./Pipfile.lock
-
-# Disable announcement plugin of jupyterlab
-RUN jupyter labextension disable "@jupyterlab/apputils-extension:announcements"
-
-# Fix permissions to support pip in Openshift environments
-RUN chmod -R g+w /opt/app-root/lib/python3.9/site-packages && \
-      fix-permissions /opt/app-root -P
+RUN echo "Installing softwares and packages" && micropipenv install && rm -f ./Pipfile.lock && \
+    # Disable announcement plugin of jupyterlab \
+    jupyter labextension disable "@jupyterlab/apputils-extension:announcements" && \
+    # Fix permissions to support pip in Openshift environments \
+    chmod -R g+w /opt/app-root/lib/python3.9/site-packages && \
+    fix-permissions /opt/app-root -P
 
 WORKDIR /opt/app-root/src
 

jupyter/minimal/ubi8-python-3.8/Dockerfile

@@ -20,8 +20,7 @@ RUN echo "Installing softwares and packages" && \
     # Replace Notebook's launcher, "(ipykernel)" with Python's version 3.x.y \
     sed -i -e "s/Python.*/$(python --version | cut -d '.' -f-2)\",/" /opt/app-root/share/jupyter/kernels/python3/kernel.json && \
     # Disable announcement plugin of jupyterlab \
-    jupyter labextension disable "@jupyterlab/apputils-extension:announcements"
-
-# Fix permissions to support pip in Openshift environments
-RUN chmod -R g+w /opt/app-root/lib/python3.9/site-packages && \
+    jupyter labextension disable "@jupyterlab/apputils-extension:announcements" && \
+    # Fix permissions to support pip in Openshift environments \
+    chmod -R g+w /opt/app-root/lib/python3.9/site-packages && \
     fix-permissions /opt/app-root -P

jupyter/minimal/ubi9-python-3.9/Dockerfile

@@ -13,11 +13,16 @@ LABEL name="odh-notebook-jupyter-rocm-pytorch-ubi9-python-3.9" \
 
 # Install Python packages and Jupyterlab extensions from Pipfile.lock
 COPY Pipfile.lock ./
+# Copy utility script
+COPY de-vendor-torch.sh ./
 
 RUN echo "Installing softwares and packages" && micropipenv install && rm -f ./Pipfile.lock && \
+    # De-vendor the ROCm libs that are embedded in Pytorch \
+    ./de-vendor-torch.sh && \
+    rm ./de-vendor-torch.sh && \
     # Replace Notebook's launcher, "(ipykernel)" with Python's version 3.x.y
     sed -i -e "s/Python.*/$(python --version | cut -d '.' -f-2)\",/" /opt/app-root/share/jupyter/kernels/python3/kernel.json && \
-    # Disable announcement plugin of jupyterlab
+    # Disable announcement plugin of jupyterlab \
     jupyter labextension disable "@jupyterlab/apputils-extension:announcements" && \
-    # Fix permissions to support pip in Openshift environments
+    # Fix permissions to support pip in Openshift environments \
     chmod -R g+w /opt/app-root/lib/python3.9/site-packages && fix-permissions /opt/app-root -P