Skip to content

Commit 57c6397

Browse files
fscnickfscnick
authored
[RayService][Test] create curl pod waiting until running (#3740)
Signed-off-by: fscnick <[email protected]>
1 parent a658405 commit 57c6397

File tree

8 files changed

+44
-75
lines changed

8 files changed

+44
-75
lines changed

‎ray-operator/test/e2erayservice/rayservice_ha_test.go

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@ import (
1010

1111
rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1"
1212
"github.com/ray-project/kuberay/ray-operator/controllers/ray/common"
13-
"github.com/ray-project/kuberay/ray-operator/test/sampleyaml"
1413
. "github.com/ray-project/kuberay/ray-operator/test/support"
1514
)
1615

@@ -268,7 +267,7 @@ func TestRayServiceGCSFaultTolerance(t *testing.T) {
268267
// Kill gcs server
269268
ExecPodCmd(test, oldHeadPod, common.RayHeadContainer, []string{"pkill", "gcs_server"})
270269
// wait for head pod not to be ready
271-
g.Eventually(HeadPod(test, rayServiceUnderlyingRayCluster), TestTimeoutShort).Should(WithTransform(sampleyaml.IsPodRunningAndReady, BeFalse()))
270+
g.Eventually(HeadPod(test, rayServiceUnderlyingRayCluster), TestTimeoutShort).Should(WithTransform(IsPodRunningAndReady, BeFalse()))
272271

273272
startTime := time.Now()
274273
// Run Locust test
@@ -285,6 +284,6 @@ func TestRayServiceGCSFaultTolerance(t *testing.T) {
285284
g.Expect(newHeadPod.Name).To(Equal(oldHeadPodName))
286285
g.Expect(newHeadPod.Status.ContainerStatuses[0].RestartCount).To(Equal(int32(1)))
287286
// Verify that all pods are running
288-
g.Expect(GetHeadPod(test, rayServiceUnderlyingRayCluster)).Should(WithTransform(sampleyaml.IsPodRunningAndReady, BeTrue()))
289-
g.Expect(GetWorkerPods(test, rayServiceUnderlyingRayCluster)).Should(WithTransform(sampleyaml.AllPodsRunningAndReady, BeTrue()))
287+
g.Expect(GetHeadPod(test, rayServiceUnderlyingRayCluster)).Should(WithTransform(IsPodRunningAndReady, BeTrue()))
288+
g.Expect(GetWorkerPods(test, rayServiceUnderlyingRayCluster)).Should(WithTransform(AllPodsRunningAndReady, BeTrue()))
290289
}

‎ray-operator/test/e2erayservice/rayservice_in_place_update_test.go

Lines changed: 3 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@ import (
1010

1111
rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1"
1212
rayv1ac "github.com/ray-project/kuberay/ray-operator/pkg/client/applyconfiguration/ray/v1"
13-
"github.com/ray-project/kuberay/ray-operator/test/sampleyaml"
1413
. "github.com/ray-project/kuberay/ray-operator/test/support"
1514
)
1615

@@ -41,14 +40,8 @@ func TestRayServiceInPlaceUpdate(t *testing.T) {
4140
curlPodName := "curl-pod"
4241
curlContainerName := "curl-container"
4342

44-
curlPod, err := CreateCurlPod(test, curlPodName, curlContainerName, namespace.Name)
43+
curlPod, err := CreateCurlPod(g, test, curlPodName, curlContainerName, namespace.Name)
4544
g.Expect(err).NotTo(HaveOccurred())
46-
// Wait until curl pod is created
47-
g.Eventually(func(g Gomega) *corev1.Pod {
48-
updatedCurlPod, err := test.Client().Core().CoreV1().Pods(curlPod.Namespace).Get(test.Ctx(), curlPod.Name, metav1.GetOptions{})
49-
g.Expect(err).NotTo(HaveOccurred())
50-
return updatedCurlPod
51-
}, TestTimeoutShort).Should(WithTransform(sampleyaml.IsPodRunningAndReady, BeTrue()))
5245

5346
LogWithTimestamp(test.T(), "Sending requests to the RayService to make sure it is ready to serve requests")
5447
stdout, _ := CurlRayServicePod(test, rayService, curlPod, curlContainerName, "/fruit", `["MANGO", 2]`)
@@ -111,14 +104,8 @@ func TestUpdateServeConfigAndRayClusterSpec(t *testing.T) {
111104
curlPodName := "curl-pod"
112105
curlContainerName := "curl-container"
113106

114-
curlPod, err := CreateCurlPod(test, curlPodName, curlContainerName, namespace.Name)
107+
curlPod, err := CreateCurlPod(g, test, curlPodName, curlContainerName, namespace.Name)
115108
g.Expect(err).NotTo(HaveOccurred())
116-
// Wait until curl pod is created
117-
g.Eventually(func(g Gomega) *corev1.Pod {
118-
updatedCurlPod, err := test.Client().Core().CoreV1().Pods(curlPod.Namespace).Get(test.Ctx(), curlPod.Name, metav1.GetOptions{})
119-
g.Expect(err).NotTo(HaveOccurred())
120-
return updatedCurlPod
121-
}, TestTimeoutShort).Should(WithTransform(sampleyaml.IsPodRunningAndReady, BeTrue()))
122109

123110
LogWithTimestamp(test.T(), "Sending requests to the RayService to make sure it is ready to serve requests")
124111
stdout, _ := CurlRayServicePod(test, rayService, curlPod, curlContainerName, "/fruit", `["MANGO", 2]`)
@@ -214,14 +201,8 @@ func TestUpdateServeConfigAndRayClusterSpecWithUpgradeDisabled(t *testing.T) {
214201
curlPodName := "curl-pod"
215202
curlContainerName := "curl-container"
216203

217-
curlPod, err := CreateCurlPod(test, curlPodName, curlContainerName, namespace.Name)
204+
curlPod, err := CreateCurlPod(g, test, curlPodName, curlContainerName, namespace.Name)
218205
g.Expect(err).NotTo(HaveOccurred())
219-
// Wait until curl pod is created
220-
g.Eventually(func(g Gomega) *corev1.Pod {
221-
updatedCurlPod, err := test.Client().Core().CoreV1().Pods(curlPod.Namespace).Get(test.Ctx(), curlPod.Name, metav1.GetOptions{})
222-
g.Expect(err).NotTo(HaveOccurred())
223-
return updatedCurlPod
224-
}, TestTimeoutShort).Should(WithTransform(sampleyaml.IsPodRunningAndReady, BeTrue()))
225206

226207
LogWithTimestamp(test.T(), "Sending requests to the RayService to make sure it is ready to serve requests")
227208
stdout, _ := CurlRayServicePod(test, rayService, curlPod, curlContainerName, "/fruit", `["MANGO", 2]`)

‎ray-operator/test/e2erayservice/rayservice_redeploy_test.go

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,10 @@ import (
44
"testing"
55

66
. "github.com/onsi/gomega"
7-
corev1 "k8s.io/api/core/v1"
87
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
98

109
"github.com/ray-project/kuberay/ray-operator/controllers/ray/utils"
1110
rayv1ac "github.com/ray-project/kuberay/ray-operator/pkg/client/applyconfiguration/ray/v1"
12-
"github.com/ray-project/kuberay/ray-operator/test/sampleyaml"
1311
. "github.com/ray-project/kuberay/ray-operator/test/support"
1412
)
1513

@@ -41,14 +39,8 @@ func TestRedeployRayServe(t *testing.T) {
4139

4240
LogWithTimestamp(test.T(), "Creating curl pod %s/%s", namespace.Name, curlPodName)
4341

44-
curlPod, err := CreateCurlPod(test, curlPodName, curlContainerName, namespace.Name)
42+
curlPod, err := CreateCurlPod(g, test, curlPodName, curlContainerName, namespace.Name)
4543
g.Expect(err).NotTo(HaveOccurred())
46-
g.Eventually(func(g Gomega) *corev1.Pod {
47-
updatedCurlPod, err := test.Client().Core().CoreV1().Pods(curlPod.Namespace).Get(test.Ctx(), curlPod.Name, metav1.GetOptions{})
48-
g.Expect(err).NotTo(HaveOccurred())
49-
return updatedCurlPod
50-
}, TestTimeoutShort).Should(WithTransform(sampleyaml.IsPodRunningAndReady, BeTrue()))
51-
LogWithTimestamp(test.T(), "Curl pod %s/%s is running and ready", namespace.Name, curlPodName)
5244

5345
LogWithTimestamp(test.T(), "Sending requests to the RayService to make sure it is ready to serve requests")
5446
stdout, _ := CurlRayServicePod(test, rayService, curlPod, curlContainerName, "/fruit", `["MANGO", 2]`)

‎ray-operator/test/e2erayservice/rayservice_upgrade_test.go

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@ import (
1212

1313
"github.com/ray-project/kuberay/ray-operator/controllers/ray/utils"
1414
rayv1ac "github.com/ray-project/kuberay/ray-operator/pkg/client/applyconfiguration/ray/v1"
15-
"github.com/ray-project/kuberay/ray-operator/test/sampleyaml"
1615
. "github.com/ray-project/kuberay/ray-operator/test/support"
1716
)
1817

@@ -44,14 +43,8 @@ func TestOldHeadPodFailDuringUpgrade(t *testing.T) {
4443

4544
LogWithTimestamp(test.T(), "Creating curl pod %s/%s", namespace.Name, curlPodName)
4645

47-
curlPod, err := CreateCurlPod(test, curlPodName, curlContainerName, namespace.Name)
46+
curlPod, err := CreateCurlPod(g, test, curlPodName, curlContainerName, namespace.Name)
4847
g.Expect(err).NotTo(HaveOccurred())
49-
g.Eventually(func(g Gomega) *corev1.Pod {
50-
updatedCurlPod, err := test.Client().Core().CoreV1().Pods(curlPod.Namespace).Get(test.Ctx(), curlPod.Name, metav1.GetOptions{})
51-
g.Expect(err).NotTo(HaveOccurred())
52-
return updatedCurlPod
53-
}, TestTimeoutShort).Should(WithTransform(sampleyaml.IsPodRunningAndReady, BeTrue()))
54-
LogWithTimestamp(test.T(), "Curl pod %s/%s is running and ready", namespace.Name, curlPodName)
5548

5649
LogWithTimestamp(test.T(), "Sending requests to the RayService to make sure it is ready to serve requests")
5750
stdout, _ := CurlRayServicePod(test, rayService, curlPod, curlContainerName, "/fruit", `["MANGO", 2]`)

‎ray-operator/test/e2eupgrade/rayservice_operator_upgrade_test.go

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@ import (
1414
"github.com/ray-project/kuberay/ray-operator/controllers/ray/utils"
1515
rayv1ac "github.com/ray-project/kuberay/ray-operator/pkg/client/applyconfiguration/ray/v1"
1616
e2e "github.com/ray-project/kuberay/ray-operator/test/e2erayservice"
17-
"github.com/ray-project/kuberay/ray-operator/test/sampleyaml"
1817
. "github.com/ray-project/kuberay/ray-operator/test/support"
1918
)
2019

@@ -57,14 +56,8 @@ func TestZeroDowntimeUpgradeAfterOperatorUpgrade(t *testing.T) {
5756
curlContainerName := "curl-container"
5857

5958
test.T().Logf("Creating curl pod %s/%s", namespace.Name, curlPodName)
60-
curlPod, err := CreateCurlPod(test, curlPodName, curlContainerName, namespace.Name)
59+
curlPod, err := CreateCurlPod(g, test, curlPodName, curlContainerName, namespace.Name)
6160
g.Expect(err).NotTo(HaveOccurred())
62-
g.Eventually(func(g Gomega) *corev1.Pod {
63-
updatedCurlPod, err := test.Client().Core().CoreV1().Pods(curlPod.Namespace).Get(test.Ctx(), curlPod.Name, metav1.GetOptions{})
64-
g.Expect(err).NotTo(HaveOccurred())
65-
return updatedCurlPod
66-
}, TestTimeoutShort).Should(WithTransform(sampleyaml.IsPodRunningAndReady, BeTrue()))
67-
test.T().Logf("Curl pod %s/%s is running and ready", namespace.Name, curlPodName)
6861

6962
// Validate RayService is able to serve requests
7063
test.T().Logf("Sending requests to the RayService to make sure it is ready to serve requests")

‎ray-operator/test/sampleyaml/support.go

Lines changed: 0 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ import (
99
. "github.com/onsi/gomega"
1010
"github.com/stretchr/testify/assert"
1111
"github.com/stretchr/testify/require"
12-
corev1 "k8s.io/api/core/v1"
1312

1413
rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1"
1514
"github.com/ray-project/kuberay/ray-operator/controllers/ray/utils"
@@ -26,27 +25,6 @@ func GetSampleYAMLDir(t Test) string {
2625
return sampleYAMLDir
2726
}
2827

29-
func IsPodRunningAndReady(pod *corev1.Pod) bool {
30-
if pod.Status.Phase != corev1.PodRunning {
31-
return false
32-
}
33-
for _, condition := range pod.Status.Conditions {
34-
if condition.Type == corev1.PodReady && condition.Status == corev1.ConditionTrue {
35-
return true
36-
}
37-
}
38-
return false
39-
}
40-
41-
func AllPodsRunningAndReady(pods []corev1.Pod) bool {
42-
for _, pod := range pods {
43-
if !IsPodRunningAndReady(&pod) {
44-
return false
45-
}
46-
}
47-
return true
48-
}
49-
5028
func SubmitJobsToAllPods(t Test, rayCluster *rayv1.RayCluster) func(Gomega) {
5129
return func(g Gomega) {
5230
pods, err := GetAllPods(t, rayCluster)

‎ray-operator/test/support/core.go

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -147,7 +147,7 @@ func SetupPortForward(t Test, podName, namespace string, localPort, remotePort i
147147
return stopChan, nil
148148
}
149149

150-
func CreateCurlPod(t Test, podName, containerName, namespace string) (*corev1.Pod, error) {
150+
func CreateCurlPod(g *gomega.WithT, t Test, podName, containerName, namespace string) (*corev1.Pod, error) {
151151
// Define the podSpec spec
152152
podSpec := &corev1.Pod{
153153
ObjectMeta: metav1.ObjectMeta{
@@ -164,5 +164,16 @@ func CreateCurlPod(t Test, podName, containerName, namespace string) (*corev1.Po
164164
},
165165
},
166166
}
167-
return t.Client().Core().CoreV1().Pods(namespace).Create(t.Ctx(), podSpec, metav1.CreateOptions{})
167+
168+
curlPod, err := t.Client().Core().CoreV1().Pods(namespace).Create(t.Ctx(), podSpec, metav1.CreateOptions{})
169+
g.Expect(err).NotTo(gomega.HaveOccurred())
170+
// Wait until curl pod is created
171+
g.Eventually(func(g gomega.Gomega) *corev1.Pod {
172+
updatedCurlPod, err := t.Client().Core().CoreV1().Pods(curlPod.Namespace).Get(t.Ctx(), curlPod.Name, metav1.GetOptions{})
173+
g.Expect(err).NotTo(gomega.HaveOccurred())
174+
return updatedCurlPod
175+
}, TestTimeoutShort).Should(gomega.WithTransform(IsPodRunningAndReady, gomega.BeTrue()))
176+
177+
LogWithTimestamp(t.T(), "Curl pod %s/%s is running and ready", curlPod.Namespace, curlPod.Name)
178+
return t.Client().Core().CoreV1().Pods(curlPod.Namespace).Get(t.Ctx(), curlPod.Name, metav1.GetOptions{})
168179
}

‎ray-operator/test/support/support.go

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77

88
"github.com/onsi/gomega"
99
"github.com/onsi/gomega/format"
10+
v1 "k8s.io/api/core/v1"
1011
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1112
)
1213

@@ -49,3 +50,24 @@ func init() {
4950
// Disable object truncation on test results
5051
format.MaxLength = 0
5152
}
53+
54+
func IsPodRunningAndReady(pod *v1.Pod) bool {
55+
if pod.Status.Phase != v1.PodRunning {
56+
return false
57+
}
58+
for _, condition := range pod.Status.Conditions {
59+
if condition.Type == v1.PodReady && condition.Status == v1.ConditionTrue {
60+
return true
61+
}
62+
}
63+
return false
64+
}
65+
66+
func AllPodsRunningAndReady(pods []v1.Pod) bool {
67+
for _, pod := range pods {
68+
if !IsPodRunningAndReady(&pod) {
69+
return false
70+
}
71+
}
72+
return true
73+
}

0 commit comments

Comments
 (0)