Fix raytune_oai_mr_grpc_test test

Author: sutaakar

go.mod

@@ -7,7 +7,7 @@ toolchain go1.21.5
 require (
 	github.com/kubeflow/training-operator v1.7.0
 	github.com/onsi/gomega v1.31.1
-	github.com/project-codeflare/codeflare-common v0.0.0-20241121090634-e99e941c6921
+	github.com/project-codeflare/codeflare-common v0.0.0-20241203135025-af256802fc2d
 	github.com/prometheus/client_golang v1.20.4
 	github.com/prometheus/common v0.57.0
 	github.com/ray-project/kuberay/ray-operator v1.1.0-alpha.0

go.sum

@@ -365,8 +365,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/project-codeflare/appwrapper v0.8.0 h1:vWHNtXUtHutN2EzYb6rryLdESnb8iDXsCokXOuNYXvg=
 github.com/project-codeflare/appwrapper v0.8.0/go.mod h1:FMQ2lI3fz6LakUVXgN1FTdpsc3BBkNIZZgtMmM9J5UM=
-github.com/project-codeflare/codeflare-common v0.0.0-20241121090634-e99e941c6921 h1:OI9jKDW4yxbXDTpf4Y+8H4uVfdCH+jIqN0JTQfdUMYw=
-github.com/project-codeflare/codeflare-common v0.0.0-20241121090634-e99e941c6921/go.mod h1:v7XKwaDoCspsHQlWJNarO7gOpR+iumSS+c1bWs3kJOI=
+github.com/project-codeflare/codeflare-common v0.0.0-20241203135025-af256802fc2d h1:WN/cN/giLiicdGjnztRYgfR7K7biaGmPO98WdWMppos=
+github.com/project-codeflare/codeflare-common v0.0.0-20241203135025-af256802fc2d/go.mod h1:v7XKwaDoCspsHQlWJNarO7gOpR+iumSS+c1bWs3kJOI=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=

tests/odh/notebook.go

@@ -29,6 +29,11 @@ import (
 	"k8s.io/apimachinery/pkg/util/yaml"
 )
 
+const (
+	NOTEBOOK_POD_NAME       = "jupyter-nb-kube-3aadmin-0"
+	NOTEBOOK_CONTAINER_NAME = "jupyter-nb-kube-3aadmin"
+)
+
 var notebookResource = schema.GroupVersionResource{Group: "kubeflow.org", Version: "v1", Resource: "notebooks"}
 
 type NotebookProps struct {
@@ -48,6 +53,7 @@ type NotebookProps struct {
 	S3BucketName              string
 	S3AccessKeyId             string
 	S3SecretAccessKey         string
+	S3Endpoint                string
 	S3DefaultRegion           string
 }
 
@@ -57,12 +63,14 @@ func createNotebook(test Test, namespace *corev1.Namespace, notebookUserToken, r
 	s3BucketName, s3BucketNameExists := GetStorageBucketName()
 	s3AccessKeyId, _ := GetStorageBucketAccessKeyId()
 	s3SecretAccessKey, _ := GetStorageBucketSecretKey()
+	s3Endpoint, _ := GetStorageBucketDefaultEndpoint()
 	s3DefaultRegion, _ := GetStorageBucketDefaultRegion()
 
 	if !s3BucketNameExists {
 		s3BucketName = "''"
 		s3AccessKeyId = "''"
 		s3SecretAccessKey = "''"
+		s3Endpoint = "''"
 		s3DefaultRegion = "''"
 	}
 
@@ -82,6 +90,7 @@ func createNotebook(test Test, namespace *corev1.Namespace, notebookUserToken, r
 		S3BucketName:              s3BucketName,
 		S3AccessKeyId:             s3AccessKeyId,
 		S3SecretAccessKey:         s3SecretAccessKey,
+		S3Endpoint:                s3Endpoint,
 		S3DefaultRegion:           s3DefaultRegion,
 		PipIndexUrl:               GetPipIndexURL(),
 		PipTrustedHost:            GetPipTrustedHost(),

tests/odh/raytune_oai_mr_grpc_test.go

@@ -23,6 +23,9 @@ import (
 	"path/filepath"
 	"strings"
 	"testing"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
 
 	. "github.com/onsi/gomega"
 	. "github.com/project-codeflare/codeflare-common/support"
@@ -88,7 +91,6 @@ func raytuneHpo(t *testing.T, numGpus int) {
 
 	// list changes required in llm-deepspeed-finetune-demo.ipynb file and update those
 	requiredChangesInNotebook := map[string]string{
-		"skip_tls=True":             "skip_tls=False",
 		"token = 'TOKEN'":           fmt.Sprintf("token='%s'", userToken),
 		"server = 'SERVER'":         fmt.Sprintf("server='%s'", GetOpenShiftApiUrl(test)),
 		"name='terrestial-raytest'": fmt.Sprintf("name='%s',\\n\",\n\t\t\"    namespace='%s'", "terrestial-raytest", namespace.Name),
@@ -132,7 +134,7 @@ func raytuneHpo(t *testing.T, numGpus int) {
 		)
 
 	// Make sure the RayCluster finishes and is deleted
-	test.Eventually(RayClusters(test, namespace.Name), TestTimeoutLong).
-		Should(BeEmpty())
+	test.Eventually(PodLog(test, namespace.Name, NOTEBOOK_POD_NAME, corev1.PodLogOptions{Container: NOTEBOOK_CONTAINER_NAME}), 20*time.Minute).
+		Should(ContainSubstring("Model Prediction:"))
 
 }

tests/odh/resources/custom-nb-small.yaml

@@ -54,6 +54,8 @@ spec:
           value: {{.S3AccessKeyId}}
         - name: AWS_SECRET_ACCESS_KEY
           value: {{.S3SecretAccessKey}}
+        - name: AWS_S3_ENDPOINT
+          value: {{.S3Endpoint}}
         - name: AWS_DEFAULT_REGION
           value: {{.S3DefaultRegion}}
         - name: AWS_S3_BUCKET