Skip to content

Commit 1264faa

Browse files
sutaakaropenshift-merge-bot[bot]
sutaakar
authored and
openshift-merge-bot[bot]
committed
Adjust existing e2e tests to provide custom Pypi index URL
1 parent d85339b commit 1264faa

File tree

12 files changed

+143
-18
lines changed

12 files changed

+143
-18
lines changed

‎README.md‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,21 @@ The e2e tests can be executed locally by running the following commands:
8686
8787
Alternatively, You can run the e2e test(s) from your IDE / debugger.
8888
89+
#### Testing on disconnected cluster
90+
91+
To properly run e2e tests on disconnected cluster user has to provide additional environment variables to properly configure testing environment:
92+
93+
- `CODEFLARE_TEST_PYTORCH_IMAGE` - image tag for image used to run training job using MCAD
94+
- `CODEFLARE_TEST_RAY_IMAGE` - image tag for Ray cluster image
95+
- `MNIST_DATASET_URL` - URL where MNIST dataset is available
96+
- `PIP_INDEX_URL` - URL where PyPI server with needed dependencies is running
97+
- `PIP_TRUSTED_HOST` - PyPI server hostname
98+
99+
For ODH tests additional environment variables are needed:
100+
101+
- `NOTEBOOK_IMAGE_STREAM_NAME` - name of the ODH Notebook ImageStream to be used
102+
- `ODH_NAMESPACE` - namespace where ODH is installed
103+
89104
## Release
90105
91106
1. Invoke [project-codeflare-release.yaml](https://github.com/project-codeflare/codeflare-operator/actions/workflows/project-codeflare-release.yml)

‎go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ go 1.20
55
require (
66
github.com/onsi/gomega v1.27.10
77
github.com/openshift/api v0.0.0-20230213134911-7ba313770556
8-
github.com/project-codeflare/codeflare-common v0.0.0-20240111082724-8f0684651717
8+
github.com/project-codeflare/codeflare-common v0.0.0-20240201153809-2e7292120303
99
github.com/project-codeflare/instascale v0.4.0
1010
github.com/project-codeflare/multi-cluster-app-dispatcher v1.39.0
1111
github.com/ray-project/kuberay/ray-operator v1.0.0

‎go.sum‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -384,8 +384,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
384384
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
385385
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
386386
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
387-
github.com/project-codeflare/codeflare-common v0.0.0-20240111082724-8f0684651717 h1:knUKEKvfEzVuSwQ4NAe2+I/Oxo4WztU5rYR8d/F66Lw=
388-
github.com/project-codeflare/codeflare-common v0.0.0-20240111082724-8f0684651717/go.mod h1:2Ck9LC+6Xi4jTDSlCJoP00tCzSrxek0roLsjvUgL2gY=
387+
github.com/project-codeflare/codeflare-common v0.0.0-20240201153809-2e7292120303 h1:30LG8751WElZmWA3mVS8l23l2oZnUCqbDkLCyy0U/p0=
388+
github.com/project-codeflare/codeflare-common v0.0.0-20240201153809-2e7292120303/go.mod h1:2Ck9LC+6Xi4jTDSlCJoP00tCzSrxek0roLsjvUgL2gY=
389389
github.com/project-codeflare/instascale v0.4.0 h1:l/cb+x4FrJ2bN9wXjv1mCngy77tVw0CLMiqJovTAflo=
390390
github.com/project-codeflare/instascale v0.4.0/go.mod h1:CpduFXKeuqYW4Ph1CPOJV6dpAdpebOxhbU4CmccZWSo=
391391
github.com/project-codeflare/multi-cluster-app-dispatcher v1.39.0 h1:zoS7pEAWK6eGELPCIIHB3W8Zb/a27Rf55ChYso7EV3o=

‎test/e2e/mnist_pytorch_mcad_job_test.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,8 @@ func TestMNISTPyTorchMCAD(t *testing.T) {
8181
Env: []corev1.EnvVar{
8282
{Name: "PYTHONUSERBASE", Value: "/workdir"},
8383
{Name: "MNIST_DATASET_URL", Value: GetMnistDatasetURL()},
84+
{Name: "PIP_INDEX_URL", Value: GetPipIndexURL()},
85+
{Name: "PIP_TRUSTED_HOST", Value: GetPipTrustedHost()},
8486
},
8587
Command: []string{"/bin/sh", "-c", "pip install -r /test/requirements.txt && torchrun /test/mnist.py"},
8688
VolumeMounts: []corev1.VolumeMount{

‎test/e2e/mnist_rayjob_mcad_raycluster_test.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -229,6 +229,8 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) {
229229
- torchvision==0.12.0
230230
env_vars:
231231
MNIST_DATASET_URL: "` + GetMnistDatasetURL() + `"
232+
PIP_INDEX_URL: "` + GetPipIndexURL() + `"
233+
PIP_TRUSTED_HOST: "` + GetPipTrustedHost() + `"
232234
`,
233235
ClusterSelector: map[string]string{
234236
RayJobDefaultClusterSelectorKey: rayCluster.Name,

‎test/odh/mcad_ray_test.go‎

Lines changed: 40 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,8 @@ func TestMCADRay(t *testing.T) {
3838
config := CreateConfigMap(test, namespace.Name, map[string][]byte{
3939
// MNIST Ray Notebook
4040
jupyterNotebookConfigMapFileName: ReadFile(test, "resources/mnist_ray_mini.ipynb"),
41-
"mnist.py": ReadFile(test, "resources/mnist.py"),
42-
"requirements.txt": ReadFile(test, "resources/requirements.txt"),
41+
"mnist.py": readMnistPy(test),
42+
"requirements.txt": readRequirementsTxt(test),
4343
})
4444

4545
// Create RBAC, retrieve token for user with limited rights
@@ -59,6 +59,11 @@ func TestMCADRay(t *testing.T) {
5959
APIGroups: []string{"route.openshift.io"},
6060
Resources: []string{"routes"},
6161
},
62+
{
63+
Verbs: []string{"get", "list"},
64+
APIGroups: []string{"networking.k8s.io"},
65+
Resources: []string{"ingresses"},
66+
},
6267
}
6368

6469
// Create cluster wide RBAC, required for SDK OpenShift check
@@ -96,3 +101,36 @@ func TestMCADRay(t *testing.T) {
96101
test.Eventually(AppWrappers(test, namespace), TestTimeoutLong).
97102
Should(HaveLen(0))
98103
}
104+
105+
func readRequirementsTxt(test Test) []byte {
106+
// Read the requirements.txt from resources and perform replacements for custom values using go template
107+
props := struct {
108+
PipIndexUrl string
109+
PipTrustedHost string
110+
}{
111+
PipIndexUrl: "--index " + GetPipIndexURL(),
112+
}
113+
114+
// Provide trusted host only if defined
115+
if len(GetPipTrustedHost()) > 0 {
116+
props.PipTrustedHost = "--trusted-host " + GetPipTrustedHost()
117+
}
118+
119+
template, err := files.ReadFile("resources/requirements.txt")
120+
test.Expect(err).NotTo(HaveOccurred())
121+
122+
return ParseTemplate(test, template, props)
123+
}
124+
125+
func readMnistPy(test Test) []byte {
126+
// Read the mnist.py from resources and perform replacements for custom values using go template
127+
props := struct {
128+
MnistDatasetURL string
129+
}{
130+
MnistDatasetURL: GetMnistDatasetURL(),
131+
}
132+
template, err := files.ReadFile("resources/mnist.py")
133+
test.Expect(err).NotTo(HaveOccurred())
134+
135+
return ParseTemplate(test, template, props)
136+
}

‎test/odh/notebook.go‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@ package odh
1818

1919
import (
2020
"bytes"
21-
"html/template"
2221

2322
gomega "github.com/onsi/gomega"
2423
. "github.com/project-codeflare/codeflare-common/support"
@@ -44,6 +43,7 @@ type NotebookProps struct {
4443
OpenDataHubNamespace string
4544
ImageStreamName string
4645
ImageStreamTag string
46+
RayImage string
4747
NotebookConfigMapName string
4848
NotebookConfigMapFileName string
4949
NotebookPVC string
@@ -66,23 +66,19 @@ func createNotebook(test Test, namespace *corev1.Namespace, notebookToken, jupyt
6666
OpenDataHubNamespace: GetOpenDataHubNamespace(),
6767
ImageStreamName: GetNotebookImageStreamName(test),
6868
ImageStreamTag: recommendedTagName,
69+
RayImage: GetRayImage(),
6970
NotebookConfigMapName: jupyterNotebookConfigMapName,
7071
NotebookConfigMapFileName: jupyterNotebookConfigMapFileName,
7172
NotebookPVC: notebookPVC.Name,
7273
}
7374
notebookTemplate, err := files.ReadFile("resources/custom-nb-small.yaml")
7475
test.Expect(err).NotTo(gomega.HaveOccurred())
75-
parsedNotebookTemplate, err := template.New("notebook").Parse(string(notebookTemplate))
76-
test.Expect(err).NotTo(gomega.HaveOccurred())
7776

78-
// Filter template and store results to the buffer
79-
notebookBuffer := new(bytes.Buffer)
80-
err = parsedNotebookTemplate.Execute(notebookBuffer, notebookProps)
81-
test.Expect(err).NotTo(gomega.HaveOccurred())
77+
parsedNotebookTemplate := ParseTemplate(test, notebookTemplate, notebookProps)
8278

8379
// Create Notebook CR
8480
notebookCR := &unstructured.Unstructured{}
85-
err = yaml.NewYAMLOrJSONDecoder(notebookBuffer, 8192).Decode(notebookCR)
81+
err = yaml.NewYAMLOrJSONDecoder(bytes.NewBuffer(parsedNotebookTemplate), 8192).Decode(notebookCR)
8682
test.Expect(err).NotTo(gomega.HaveOccurred())
8783
_, err = test.Client().Dynamic().Resource(notebookResource).Namespace(namespace.Name).Create(test.Ctx(), notebookCR, metav1.CreateOptions{})
8884
test.Expect(err).NotTo(gomega.HaveOccurred())

‎test/odh/resources/custom-nb-small.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ spec:
5555
- name: OCP_TOKEN
5656
value: {{.KubernetesBearerToken}}
5757
image: image-registry.openshift-image-registry.svc:5000/{{.OpenDataHubNamespace}}/{{.ImageStreamName}}:{{.ImageStreamTag}}
58-
command: ["/bin/sh", "-c", "pip install papermill && oc login --token=${OCP_TOKEN} --server=${OCP_SERVER} --insecure-skip-tls-verify=true && papermill /opt/app-root/notebooks/{{.NotebookConfigMapFileName}} /opt/app-root/src/mcad-out.ipynb -p namespace {{.Namespace}} && sleep infinity"]
58+
command: ["/bin/sh", "-c", "pip install papermill && oc login --token=${OCP_TOKEN} --server=${OCP_SERVER} --insecure-skip-tls-verify=true && papermill /opt/app-root/notebooks/{{.NotebookConfigMapFileName}} /opt/app-root/src/mcad-out.ipynb -p namespace {{.Namespace}} -p ray_image {{.RayImage}} && sleep infinity"]
5959
# args: ["pip install papermill && oc login --token=${OCP_TOKEN} --server=${OCP_SERVER} --insecure-skip-tls-verify=true && papermill /opt/app-root/notebooks/mcad.ipynb /opt/app-root/src/mcad-out.ipynb" ]
6060
imagePullPolicy: Always
6161
# livenessProbe:

‎test/odh/resources/mnist.py‎

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
import os
1616

1717
import torch
18+
import requests
1819
from pytorch_lightning import LightningModule, Trainer
1920
from pytorch_lightning.callbacks.progress import TQDMProgressBar
2021
from torch import nn
@@ -32,6 +33,8 @@
3233
print("MASTER_ADDR: is ", os.getenv("MASTER_ADDR"))
3334
print("MASTER_PORT: is ", os.getenv("MASTER_PORT"))
3435

36+
MNIST_DATASET_URL = "{{.MnistDatasetURL}}"
37+
print("MNIST_DATASET_URL: is ", MNIST_DATASET_URL)
3538

3639
class LitMNIST(LightningModule):
3740
def __init__(self, data_dir=PATH_DATASETS, hidden_size=64, learning_rate=2e-4):
@@ -110,8 +113,34 @@ def configure_optimizers(self):
110113
####################
111114

112115
def prepare_data(self):
113-
# download
114-
print("Downloading MNIST dataset...")
116+
datasetFiles = [
117+
"t10k-images-idx3-ubyte",
118+
"t10k-labels-idx1-ubyte",
119+
"train-images-idx3-ubyte",
120+
"train-labels-idx1-ubyte"
121+
]
122+
123+
# Create required folder structure
124+
downloadLocation = os.path.join(PATH_DATASETS, "MNIST", "raw")
125+
os.makedirs(downloadLocation, exist_ok=True)
126+
print(f"{downloadLocation} folder_path created!")
127+
128+
for file in datasetFiles:
129+
print(f"Downloading MNIST dataset {file}... to path : {downloadLocation}")
130+
response = requests.get(f"{MNIST_DATASET_URL}{file}", stream=True)
131+
filePath = os.path.join(downloadLocation, file)
132+
133+
#to download dataset file
134+
try:
135+
if response.status_code == 200:
136+
open(filePath, 'wb').write(response.content)
137+
print(f"{file}: Downloaded and saved zipped file to path - {filePath}")
138+
else:
139+
print(f"Failed to download file {file}")
140+
except Exception as e:
141+
print(e)
142+
print(f"Downloaded MNIST dataset to... {downloadLocation}")
143+
115144
MNIST(self.data_dir, train=True, download=True)
116145
MNIST(self.data_dir, train=False, download=True)
117146

‎test/odh/resources/mnist_ray_mini.ipynb‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,8 @@
2727
"outputs": [],
2828
"source": [
2929
"#parameters\n",
30-
"namespace = \"default\""
30+
"namespace = \"default\"\n",
31+
"ray_image = \"has to be specified\""
3132
]
3233
},
3334
{
@@ -40,7 +41,7 @@
4041
"outputs": [],
4142
"source": [
4243
"# Create our cluster and submit appwrapper\n",
43-
"cluster = Cluster(ClusterConfiguration(namespace=namespace, name='mnisttest', head_cpus=1, head_memory=2, num_workers=1, min_cpus=1, max_cpus=1, min_memory=1, max_memory=2, num_gpus=0, instascale=False))"
44+
"cluster = Cluster(ClusterConfiguration(namespace=namespace, name='mnisttest', head_cpus=1, head_memory=2, num_workers=1, min_cpus=1, max_cpus=1, min_memory=1, max_memory=2, num_gpus=0, instascale=False, image=ray_image))"
4445
]
4546
},
4647
{

0 commit comments

Comments
 (0)