Skip to content

Commit 2929048

Browse files
CFSNMCFSNM
committed
Merge remote-tracking branch 'upstream/main'
2 parents f8212e7 + 3f7312e commit 2929048

12 files changed

+92
-15
lines changed

‎tests/common/support/client.go

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222

2323
"k8s.io/client-go/dynamic"
2424
"k8s.io/client-go/kubernetes"
25+
storageclient "k8s.io/client-go/kubernetes/typed/storage/v1"
2526
_ "k8s.io/client-go/plugin/pkg/client/auth"
2627
"k8s.io/client-go/rest"
2728
"k8s.io/client-go/tools/clientcmd"
@@ -43,6 +44,7 @@ type Client interface {
4344
Image() imagev1.Interface
4445
Ray() rayclient.Interface
4546
Dynamic() dynamic.Interface
47+
Storage() storageclient.StorageV1Interface
4648
}
4749

4850
type testClient struct {
@@ -54,6 +56,7 @@ type testClient struct {
5456
image imagev1.Interface
5557
ray rayclient.Interface
5658
dynamic dynamic.Interface
59+
storage storageclient.StorageV1Interface
5760
}
5861

5962
var _ Client = (*testClient)(nil)
@@ -90,6 +93,10 @@ func (t *testClient) Dynamic() dynamic.Interface {
9093
return t.dynamic
9194
}
9295

96+
func (t *testClient) Storage() storageclient.StorageV1Interface {
97+
return t.storage
98+
}
99+
93100
func newTestClient(cfg *rest.Config) (Client, *rest.Config, error) {
94101
var err error
95102
if cfg == nil {
@@ -142,6 +149,11 @@ func newTestClient(cfg *rest.Config) (Client, *rest.Config, error) {
142149
return nil, nil, err
143150
}
144151

152+
storageClient, err := storageclient.NewForConfig(cfg)
153+
if err != nil {
154+
return nil, nil, err
155+
}
156+
145157
return &testClient{
146158
core: kubeClient,
147159
kubeflow: kubeflowClient,
@@ -151,5 +163,6 @@ func newTestClient(cfg *rest.Config) (Client, *rest.Config, error) {
151163
image: imageClient,
152164
ray: rayClient,
153165
dynamic: dynamicClient,
166+
storage: storageClient,
154167
}, cfg, nil
155168
}

‎tests/common/support/core.go

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -223,7 +223,23 @@ func GetServiceAccounts(t Test, namespace string) []*corev1.ServiceAccount {
223223
return ServiceAccounts(t, namespace)(t)
224224
}
225225

226-
func CreatePersistentVolumeClaim(t Test, namespace string, storageSize string, accessMode ...corev1.PersistentVolumeAccessMode) *corev1.PersistentVolumeClaim {
226+
type PVCOption Option[*corev1.PersistentVolumeClaim]
227+
228+
func StorageClassName(name string) PVCOption {
229+
return ErrorOption[*corev1.PersistentVolumeClaim](func(pvc *corev1.PersistentVolumeClaim) error {
230+
pvc.Spec.StorageClassName = &name
231+
return nil
232+
})
233+
}
234+
235+
func AccessModes(accessModes ...corev1.PersistentVolumeAccessMode) PVCOption {
236+
return ErrorOption[*corev1.PersistentVolumeClaim](func(pvc *corev1.PersistentVolumeClaim) error {
237+
pvc.Spec.AccessModes = accessModes
238+
return nil
239+
})
240+
}
241+
242+
func CreatePersistentVolumeClaim(t Test, namespace string, storageSize string, opts ...PVCOption) *corev1.PersistentVolumeClaim {
227243
t.T().Helper()
228244

229245
pvc := &corev1.PersistentVolumeClaim{
@@ -236,14 +252,22 @@ func CreatePersistentVolumeClaim(t Test, namespace string, storageSize string, a
236252
Namespace: namespace,
237253
},
238254
Spec: corev1.PersistentVolumeClaimSpec{
239-
AccessModes: accessMode,
255+
// AccessModes and StorageClassName will be set by applying options
240256
Resources: corev1.VolumeResourceRequirements{
241257
Requests: corev1.ResourceList{
242258
corev1.ResourceStorage: resource.MustParse(storageSize),
243259
},
244260
},
245261
},
246262
}
263+
264+
// Apply all provided options
265+
for _, opt := range opts {
266+
if err := opt.ApplyTo(pvc); err != nil {
267+
t.T().Fatalf("Error applying PVC option: %v", err)
268+
}
269+
}
270+
247271
pvc, err := t.Client().Core().CoreV1().PersistentVolumeClaims(namespace).Create(t.Ctx(), pvc, metav1.CreateOptions{})
248272
t.Expect(err).NotTo(gomega.HaveOccurred())
249273
t.T().Logf("Created PersistentVolumeClaim %s/%s successfully", pvc.Namespace, pvc.Name)

‎tests/common/support/storage.go

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
package support
2+
3+
import (
4+
"fmt"
5+
6+
"github.com/onsi/gomega"
7+
8+
storagev1 "k8s.io/api/storage/v1"
9+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
10+
)
11+
12+
var rwxSupportedProvisioners = map[string]bool{
13+
"nfs.csi.k8s.io": true,
14+
}
15+
16+
func GetStorageClasses(t Test) []storagev1.StorageClass {
17+
t.T().Helper()
18+
scList, err := t.Client().Storage().StorageClasses().List(t.Ctx(), metav1.ListOptions{})
19+
t.Expect(err).NotTo(gomega.HaveOccurred())
20+
return scList.Items
21+
}
22+
23+
func GetRWXStorageClass(t Test) (*storagev1.StorageClass, error) {
24+
t.T().Helper()
25+
26+
storageClasses := GetStorageClasses(t)
27+
28+
for _, sc := range storageClasses {
29+
if rwxSupportedProvisioners[sc.Provisioner] {
30+
t.T().Logf("Found StorageClass '%s' with provisioner '%s' which is likely to support RWX.", sc.Name, sc.Provisioner)
31+
return &sc, nil
32+
}
33+
}
34+
35+
return nil, fmt.Errorf("no StorageClass found that is known to support ReadWriteMany (RWX) access mode")
36+
}

‎tests/fms/kfto_kueue_sft_GPU_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,7 @@ func runMultiGpuPytorchjob(t *testing.T, modelConfigFile string, numberOfGpus in
115115
defer test.Client().Core().CoreV1().ConfigMaps(namespace).Delete(test.Ctx(), config.Name, *metav1.NewDeleteOptions(0))
116116

117117
// Create PVC for trained model
118-
outputPvc := CreatePersistentVolumeClaim(test, namespace, "200Gi", corev1.ReadWriteOnce)
118+
outputPvc := CreatePersistentVolumeClaim(test, namespace, "200Gi", AccessModes(corev1.ReadWriteOnce))
119119
defer test.Client().Core().CoreV1().PersistentVolumeClaims(namespace).Delete(test.Ctx(), outputPvc.Name, metav1.DeleteOptions{})
120120

121121
// Create training PyTorch job

‎tests/fms/kfto_kueue_sft_test.go

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -87,11 +87,11 @@ func runPytorchjobWithSFTtrainer(t *testing.T, modelConfigFile string) {
8787
defer test.Client().Kueue().KueueV1beta1().LocalQueues(namespace).Delete(test.Ctx(), localQueue.Name, metav1.DeleteOptions{})
8888

8989
// Create PVC for base model
90-
baseModelPvc := CreatePersistentVolumeClaim(test, namespace, "10Gi", corev1.ReadWriteOnce)
90+
baseModelPvc := CreatePersistentVolumeClaim(test, namespace, "10Gi", AccessModes(corev1.ReadWriteOnce))
9191
defer test.Client().Core().CoreV1().PersistentVolumeClaims(namespace).Delete(test.Ctx(), baseModelPvc.Name, metav1.DeleteOptions{})
9292

9393
// Create PVC for trained model
94-
outputPvc := CreatePersistentVolumeClaim(test, namespace, "10Gi", corev1.ReadWriteOnce)
94+
outputPvc := CreatePersistentVolumeClaim(test, namespace, "10Gi", AccessModes(corev1.ReadWriteOnce))
9595
defer test.Client().Core().CoreV1().PersistentVolumeClaims(namespace).Delete(test.Ctx(), outputPvc.Name, metav1.DeleteOptions{})
9696

9797
// Load training job config file
@@ -183,7 +183,7 @@ func TestPytorchjobUsingKueueQuota(t *testing.T) {
183183
localQueue := CreateKueueLocalQueue(test, namespace, clusterQueue.Name, AsDefaultQueue)
184184

185185
// Create PVC for base model
186-
baseModelPvc := CreatePersistentVolumeClaim(test, namespace, "10Gi", corev1.ReadWriteOnce)
186+
baseModelPvc := CreatePersistentVolumeClaim(test, namespace, "10Gi", AccessModes(corev1.ReadWriteOnce))
187187
defer test.Client().Core().CoreV1().PersistentVolumeClaims(namespace).Delete(test.Ctx(), baseModelPvc.Name, metav1.DeleteOptions{})
188188

189189
// Load training job config file
@@ -208,7 +208,7 @@ func TestPytorchjobUsingKueueQuota(t *testing.T) {
208208
config := CreateConfigMap(test, namespace, configData)
209209

210210
// Create first PVC for trained model
211-
outputPvc := CreatePersistentVolumeClaim(test, namespace, "10Gi", corev1.ReadWriteOnce)
211+
outputPvc := CreatePersistentVolumeClaim(test, namespace, "10Gi", AccessModes(corev1.ReadWriteOnce))
212212
defer test.Client().Core().CoreV1().PersistentVolumeClaims(namespace).Delete(test.Ctx(), outputPvc.Name, metav1.DeleteOptions{})
213213

214214
// Create first training PyTorch job
@@ -219,7 +219,7 @@ func TestPytorchjobUsingKueueQuota(t *testing.T) {
219219
Should(WithTransform(PyTorchJobConditionRunning, Equal(corev1.ConditionTrue)))
220220

221221
// Create second PVC for trained model
222-
secondOutputPvc := CreatePersistentVolumeClaim(test, namespace, "10Gi", corev1.ReadWriteOnce)
222+
secondOutputPvc := CreatePersistentVolumeClaim(test, namespace, "10Gi", AccessModes(corev1.ReadWriteOnce))
223223
defer test.Client().Core().CoreV1().PersistentVolumeClaims(namespace).Delete(test.Ctx(), outputPvc.Name, metav1.DeleteOptions{})
224224

225225
// Create second training PyTorch job

‎tests/kfto/kfto_mnist_sdk_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,7 @@ func runMnistSDK(t *testing.T, trainingImage string) {
111111
jupyterNotebookConfigMapFileName, namespace.Name, GetOpenShiftApiUrl(test), userToken, 0, trainingImage)}
112112

113113
// Create PVC for Notebook
114-
notebookPVC := CreatePersistentVolumeClaim(test, namespace.Name, "10Gi", corev1.ReadWriteOnce)
114+
notebookPVC := CreatePersistentVolumeClaim(test, namespace.Name, "10Gi", AccessModes(corev1.ReadWriteOnce))
115115

116116
// Create Notebook CR
117117
CreateNotebook(test, namespace, userToken, notebookCommand, config.Name, jupyterNotebookConfigMapFileName, 0, notebookPVC)

‎tests/kfto/kfto_sft_llm_test.go

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,8 +59,12 @@ func kftoSftLlm(t *testing.T, modelName string) {
5959
// Create role binding with Namespace specific admin cluster role
6060
CreateUserRoleBindingWithClusterRole(test, userName, namespace.Name, "admin")
6161

62+
// Get storageclass that supports RWX PVC provisioning
63+
storageClass, err := GetRWXStorageClass(test)
64+
test.Expect(err).ToNot(HaveOccurred(), "Failed to find an RWX supporting StorageClass")
65+
6266
// Create PVC for Notebook
63-
notebookPVC := CreatePersistentVolumeClaim(test, namespace.Name, "500Gi", corev1.ReadWriteMany)
67+
notebookPVC := CreatePersistentVolumeClaim(test, namespace.Name, "500Gi", AccessModes(corev1.ReadWriteMany), StorageClassName(storageClass.Name))
6468

6569
// Read and update the notebook content
6670
notebookContent := odh.ReadFileExt(test, workingDirectory+"/../../examples/kfto-sft-llm/sft.ipynb")

‎tests/kfto/kfto_training_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -173,7 +173,7 @@ func runKFTOPyTorchJob(t *testing.T, image string, gpu Accelerator, numGpus, num
173173
config := CreateConfigMap(test, namespace, configData)
174174

175175
// Create PVC for trained model
176-
outputPvc := CreatePersistentVolumeClaim(test, namespace, "10Gi", corev1.ReadWriteOnce)
176+
outputPvc := CreatePersistentVolumeClaim(test, namespace, "10Gi", AccessModes(corev1.ReadWriteOnce))
177177
defer test.Client().Core().CoreV1().PersistentVolumeClaims(namespace).Delete(test.Ctx(), outputPvc.Name, metav1.DeleteOptions{})
178178

179179
// Create training PyTorch job

‎tests/odh/mnist_ray_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,7 @@ func mnistRay(t *testing.T, numGpus int, gpuResourceName string, rayImage string
127127
CreateUserRoleBindingWithClusterRole(test, userName, namespace.Name, "admin")
128128

129129
// Create PVC for Notebook
130-
notebookPVC := CreatePersistentVolumeClaim(test, namespace.Name, "10Gi", corev1.ReadWriteOnce)
130+
notebookPVC := CreatePersistentVolumeClaim(test, namespace.Name, "10Gi", AccessModes(corev1.ReadWriteOnce))
131131

132132
notebookCommand := getNotebookCommand(rayImage)
133133
// Create Notebook CR

‎tests/odh/mnist_raytune_hpo_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,7 @@ func mnistRayTuneHpo(t *testing.T, numGpus int) {
111111
notebookCommand := getNotebookCommand(rayImage)
112112

113113
// Create PVC for Notebook
114-
notebookPVC := CreatePersistentVolumeClaim(test, namespace.Name, "10Gi", corev1.ReadWriteOnce)
114+
notebookPVC := CreatePersistentVolumeClaim(test, namespace.Name, "10Gi", AccessModes(corev1.ReadWriteOnce))
115115

116116
// Create Notebook CR
117117
CreateNotebook(test, namespace, userToken, notebookCommand, config.Name, jupyterNotebookConfigMapFileName, numGpus, notebookPVC)

0 commit comments

Comments
 (0)