Add test coverage for custom ray image

Author: ChughShilpa

tests/odh/mnist_ray_test.go

@@ -19,6 +19,8 @@ package odh
 import (
 	"bytes"
 	"fmt"
+	"log"
+	"os/exec"
 	"testing"
 	"time"
 
@@ -33,19 +35,50 @@ import (
 )
 
 func TestMnistRayCpu(t *testing.T) {
-	mnistRay(t, 0)
+	mnistDefaultRayImage(t, 0)
 }
 
 func TestMnistRayGpu(t *testing.T) {
-	mnistRay(t, 1)
+	mnistDefaultRayImage(t, 1)
 }
 
-func mnistRay(t *testing.T, numGpus int) {
+func TestMnistCustomRayImageCpu(t *testing.T) {
+	mnistCustomRayImage(t, 0)
+}
+
+func TestMnistCustomRayImageGpu(t *testing.T) {
+	mnistCustomRayImage(t, 1)
+}
+
+func mnistDefaultRayImage(t *testing.T, numGpus int) {
 	test := With(t)
 
 	// Create a namespace
 	namespace := test.NewTestNamespace()
 
+	// Get ray image
+	rayImage := GetRayImage()
+
+	mnistRay(test, numGpus, namespace, rayImage)
+}
+
+func mnistCustomRayImage(t *testing.T, numGpus int) {
+	test := With(t)
+
+	// Create a namespace
+	namespace := test.NewTestNamespace()
+
+	// Build and Push custom ray image
+	image := "ray-torch"
+	buildAndPushRayImage(test, namespace.Name, image)
+
+	// Get custom ray image
+	rayImage := getCustomRayImage(test, namespace.Name, image)
+
+	mnistRay(test, numGpus, namespace, rayImage)
+}
+
+func mnistRay(test Test, numGpus int, namespace *corev1.Namespace, rayImage string) {
 	// Create Kueue resources
 	resourceFlavor := CreateKueueResourceFlavor(test, v1beta1.ResourceFlavorSpec{})
 	defer test.Client().Kueue().KueueV1beta1().ResourceFlavors().Delete(test.Ctx(), resourceFlavor.Name, metav1.DeleteOptions{})
@@ -103,7 +136,7 @@ func mnistRay(t *testing.T, numGpus int) {
 	CreateUserRoleBindingWithClusterRole(test, userName, namespace.Name, "admin")
 
 	// Create Notebook CR
-	createNotebook(test, namespace, userToken, config.Name, jupyterNotebookConfigMapFileName, numGpus)
+	createNotebook(test, namespace, userToken, rayImage, config.Name, jupyterNotebookConfigMapFileName, numGpus)
 
 	// Gracefully cleanup Notebook
 	defer func() {
@@ -221,3 +254,25 @@ func readMnistScriptTemplate(test Test, filePath string) []byte {
 
 	return ParseTemplate(test, template, props)
 }
+
+func buildAndPushRayImage(test Test, namespace string, image string) {
+	cmd := exec.Command("resources/custom_image.sh", namespace, image, "-c", "echo stdout; echo 1>&2 stderr")
+
+	stdoutStderr, err := cmd.CombinedOutput()
+	if err != nil {
+		log.Fatal("Error executing custom_image script :", err)
+	}
+	test.Expect(err).NotTo(HaveOccurred())
+
+	fmt.Printf("Logs of build and custom ray image . . .\n %s", stdoutStderr)
+}
+
+func getCustomRayImage(test Test, namespace string, image string) string {
+	tag := "latest"
+	name := image + ":" + tag
+
+	imageStreamTag := GetImageStreamTag(test, namespace, name)
+	imageReference := imageStreamTag.Image.DockerImageReference
+
+	return imageReference
+}

tests/odh/mnist_raytune_hpo_test.go

@@ -103,8 +103,11 @@ func mnistRayTuneHpo(t *testing.T, numGpus int) {
 	// Create role binding with Namespace specific admin cluster role
 	CreateUserRoleBindingWithClusterRole(test, userName, namespace.Name, "admin")
 
+	// Get ray image
+	rayImage := GetRayImage()
+
 	// Create Notebook CR
-	createNotebook(test, namespace, userToken, config.Name, jupyterNotebookConfigMapFileName, numGpus)
+	createNotebook(test, namespace, userToken, rayImage, config.Name, jupyterNotebookConfigMapFileName, numGpus)
 
 	// Gracefully cleanup Notebook
 	defer func() {

tests/odh/notebook.go

@@ -51,7 +51,7 @@ type NotebookProps struct {
 	S3DefaultRegion           string
 }
 
-func createNotebook(test Test, namespace *corev1.Namespace, notebookUserToken, jupyterNotebookConfigMapName, jupyterNotebookConfigMapFileName string, numGpus int) {
+func createNotebook(test Test, namespace *corev1.Namespace, notebookUserToken, rayImage string, jupyterNotebookConfigMapName, jupyterNotebookConfigMapFileName string, numGpus int) {
 	// Create PVC for Notebook
 	notebookPVC := CreatePersistentVolumeClaim(test, namespace.Name, "10Gi", corev1.ReadWriteOnce)
 	s3BucketName, s3BucketNameExists := GetStorageBucketName()
@@ -73,7 +73,7 @@ func createNotebook(test Test, namespace *corev1.Namespace, notebookUserToken, j
 		KubernetesUserBearerToken: notebookUserToken,
 		Namespace:                 namespace.Name,
 		OpenDataHubNamespace:      GetOpenDataHubNamespace(test),
-		RayImage:                  GetRayImage(),
+		RayImage:                  rayImage,
 		NotebookImage:             GetNotebookImage(test),
 		NotebookConfigMapName:     jupyterNotebookConfigMapName,
 		NotebookConfigMapFileName: jupyterNotebookConfigMapFileName,

tests/odh/ray_finetune_llm_deepspeed_test.go

@@ -99,8 +99,11 @@ func rayFinetuneLlmDeepspeed(t *testing.T, numGpus int, modelName string, modelC
 
 	config := CreateConfigMap(test, namespace.Name, configMap)
 
+	// Get ray image
+	rayImage := GetRayImage()
+
 	// Create Notebook CR
-	createNotebook(test, namespace, userToken, config.Name, jupyterNotebookConfigMapFileName, numGpus)
+	createNotebook(test, namespace, userToken, rayImage, config.Name, jupyterNotebookConfigMapFileName, numGpus)
 
 	// Gracefully cleanup Notebook
 	defer func() {

tests/odh/resources/custom_image.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+namespace=$1
+image=$2
+
+echo "switch to current project . . ."
+oc project $namespace
+
+echo "Build custom container image using podman . . ."
+cd ./../../images/runtime/examples
+cd $image
+podman build -t $image -f Dockerfile
+
+echo "Expose the integrated container registry . . ."
+oc patch configs.imageregistry.operator.openshift.io/cluster --patch '{"spec":{"defaultRoute":true}}' --type=merge
+
+echo "Wait for the route to the container registry to be admitted . . ."
+oc wait -n openshift-image-registry route/default-route --for=jsonpath='{.status.ingress[0].conditions[0].status}'=True
+
+echo "Login to the container registry . . ."
+podman login -u $(oc whoami) -p $(oc whoami -t) $(oc registry info)
+
+echo "Push the image to the integrated container registry . . ."
+podman tag $image $(oc registry info)/$namespace/$image
+podman push $(oc registry info)/$namespace/$image
+
+echo "Custom Ray Image is . . . "
+oc get is $image -o jsonpath='{.status.tags[?(@.tag=="latest")].items[0].dockerImageReference}'