Remove GPU utilization verification

Author: sutaakar

tests/kfto/kfto_training_test.go

@@ -19,7 +19,6 @@ package kfto
 import (
 	"fmt"
 	"testing"
-	"time"
 
 	kftov1 "github.com/kubeflow/training-operator/pkg/apis/kubeflow.org/v1"
 	. "github.com/onsi/gomega"
@@ -183,27 +182,6 @@ func runKFTOPyTorchJob(t *testing.T, image string, gpu Accelerator, numGpus, num
 	test.Eventually(PyTorchJob(test, namespace, tuningJob.Name), TestTimeoutDouble).
 		Should(WithTransform(PyTorchJobConditionRunning, Equal(corev1.ConditionTrue)))
 
-	// Verify GPU utilization
-	if IsOpenShift(test) && gpu == NVIDIA {
-		trainingPods := GetPods(test, namespace, metav1.ListOptions{LabelSelector: "training.kubeflow.org/job-name=" + tuningJob.GetName()})
-		test.Expect(trainingPods).To(HaveLen(numberOfWorkerNodes + 1)) // +1 is a master node
-
-		for _, trainingPod := range trainingPods {
-			// Check that GPUs for training pods were utilized recently
-			test.Eventually(OpenShiftPrometheusGpuUtil(test, trainingPod, gpu), 10*time.Minute).
-				Should(
-					And(
-						HaveLen(numGpus),
-						ContainElement(
-							// Check that at least some GPU was utilized on more than 10%
-							HaveField("Value", BeNumerically(">", 10)),
-						),
-					),
-				)
-		}
-		test.T().Log("All GPUs were successfully utilized")
-	}
-
 	// Make sure the PyTorch job succeeded
 	test.Eventually(PyTorchJob(test, namespace, tuningJob.Name), TestTimeoutLong).Should(WithTransform(PyTorchJobConditionSucceeded, Equal(corev1.ConditionTrue)))
 	test.T().Logf("PytorchJob %s/%s ran successfully", tuningJob.Namespace, tuningJob.Name)