Merge remote-tracking branch 'upstream/main'

Author: m-rafeeq

tests/common/environment.go

@@ -19,7 +19,9 @@ package common
 import (
 	"flag"
 	"os"
+	"os/exec"
 	"slices"
+	"strings"
 
 	. "github.com/opendatahub-io/distributed-workloads/tests/common/support"
 )
@@ -81,6 +83,49 @@ func GetNotebookUserPassword(t Test) string {
 	return password
 }
 
+// GenerateNotebookUserToken generates an OpenShift token using oc login with username and password
+func GenerateNotebookUserToken(t Test) string {
+	userName := GetNotebookUserName(t)
+	password := GetNotebookUserPassword(t)
+
+	// Use own kubeconfig file to retrieve user token to keep it separated from main test credentials
+	tempFile, err := os.CreateTemp("", "custom-kubeconfig-")
+	if err != nil {
+		t.T().Fatalf("Failed to create temp kubeconfig file: %v", err)
+	}
+	defer os.Remove(tempFile.Name())
+
+	// Login by oc CLI using username and password
+	cmd := exec.Command("oc", "login", "-u", userName, "-p", password, GetOpenShiftApiUrl(t), "--insecure-skip-tls-verify=true", "--kubeconfig="+tempFile.Name())
+	out, err := cmd.Output()
+	if err != nil {
+		if exitError, ok := err.(*exec.ExitError); ok {
+			t.T().Logf("Error running 'oc login' command: %v\n", exitError)
+			t.T().Logf("Output: %s\n", out)
+			t.T().Logf("Error output: %s\n", exitError.Stderr)
+		} else {
+			t.T().Logf("Error running 'oc login' command: %v\n", err)
+		}
+		t.T().FailNow()
+	}
+
+	// Use oc CLI to retrieve user token from kubeconfig
+	cmd = exec.Command("oc", "whoami", "--show-token", "--kubeconfig="+tempFile.Name())
+	out, err = cmd.Output()
+	if err != nil {
+		if exitError, ok := err.(*exec.ExitError); ok {
+			t.T().Logf("Error running 'oc whoami' command: %v\n", exitError)
+			t.T().Logf("Output: %s\n", out)
+			t.T().Logf("Error output: %s\n", exitError.Stderr)
+		} else {
+			t.T().Logf("Error running 'oc whoami' command: %v\n", err)
+		}
+		t.T().FailNow()
+	}
+
+	return strings.TrimSpace(string(out))
+}
+
 func GetNotebookImage(t Test) string {
 	notebook_image, ok := os.LookupEnv(notebookImage)
 	if !ok {

tests/trainer/kubeflow_sdk_test.go

@@ -40,3 +40,51 @@ func TestSftTrainingHubMultiNodeMultiGPU(t *testing.T) {
 	Tags(t, KftoCuda, MultiNodeMultiGpu(2, support.NVIDIA, 1))
 	sdktests.RunSftTrainingHubMultiGpuDistributedTraining(t)
 }
+
+// CPU tests
+func TestRhaiTrainingProgressionCPU(t *testing.T) {
+	Tags(t, Tier1)
+	sdktests.RunRhaiFeaturesProgressionTest(t, support.CPU)
+}
+
+func TestRhaiJitCheckpointingCPU(t *testing.T) {
+	Tags(t, Tier1)
+	sdktests.RunRhaiFeaturesCheckpointTest(t, support.CPU)
+}
+
+func TestRhaiFeaturesCPU(t *testing.T) {
+	Tags(t, Tier1)
+	sdktests.RunRhaiFeaturesAllTest(t, support.CPU)
+}
+
+// CUDA (NVIDIA) GPU tests - 2 nodes, 1 GPU each
+func TestRhaiTrainingProgressionCuda(t *testing.T) {
+	Tags(t, KftoCuda, MultiNodeGpu(2, support.NVIDIA))
+	sdktests.RunRhaiFeaturesProgressionTest(t, support.NVIDIA)
+}
+
+func TestRhaiJitCheckpointingCuda(t *testing.T) {
+	Tags(t, KftoCuda, MultiNodeGpu(2, support.NVIDIA))
+	sdktests.RunRhaiFeaturesCheckpointTest(t, support.NVIDIA)
+}
+
+func TestRhaiFeaturesCuda(t *testing.T) {
+	Tags(t, KftoCuda, MultiNodeGpu(2, support.NVIDIA))
+	sdktests.RunRhaiFeaturesAllTest(t, support.NVIDIA)
+}
+
+// ROCm (AMD) GPU tests - 2 nodes, 1 GPU each
+func TestRhaiTrainingProgressionRocm(t *testing.T) {
+	Tags(t, KftoRocm, MultiNodeGpu(2, support.AMD))
+	sdktests.RunRhaiFeaturesProgressionTest(t, support.AMD)
+}
+
+func TestRhaiJitCheckpointingRocm(t *testing.T) {
+	Tags(t, KftoRocm, MultiNodeGpu(2, support.AMD))
+	sdktests.RunRhaiFeaturesCheckpointTest(t, support.AMD)
+}
+
+func TestRhaiFeaturesRocm(t *testing.T) {
+	Tags(t, KftoRocm, MultiNodeGpu(2, support.AMD))
+	sdktests.RunRhaiFeaturesAllTest(t, support.AMD)
+}