Skip to content

Commit 9b6b053

Browse files
pawelpaszkipawelpaszki
authored
test: add worker pod to upgrade tests for stability (#3891)
1 parent 8b47826 commit 9b6b053

File tree

3 files changed

+105
-6
lines changed

3 files changed

+105
-6
lines changed

‎ray-operator/test/e2erayservice/support.go

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -177,8 +177,7 @@ func RayServiceSampleYamlApplyConfiguration() *rayv1ac.RayServiceSpecApplyConfig
177177
WithLimits(corev1.ResourceList{
178178
corev1.ResourceCPU: resource.MustParse("2"),
179179
corev1.ResourceMemory: resource.MustParse("3Gi"),
180-
})))))),
181-
)
180+
})))))))
182181
}
183182

184183
func waitingForRayClusterSwitch(g *WithT, test Test, rayService *rayv1.RayService, oldRayClusterName string) {

‎ray-operator/test/e2eupgrade/rayservice_operator_upgrade_test.go

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@ import (
1313
rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1"
1414
"github.com/ray-project/kuberay/ray-operator/controllers/ray/utils"
1515
rayv1ac "github.com/ray-project/kuberay/ray-operator/pkg/client/applyconfiguration/ray/v1"
16-
e2e "github.com/ray-project/kuberay/ray-operator/test/e2erayservice"
1716
. "github.com/ray-project/kuberay/ray-operator/test/support"
1817
)
1918

@@ -36,7 +35,7 @@ func TestZeroDowntimeUpgradeAfterOperatorUpgrade(t *testing.T) {
3635
test.T().Logf("Detected upgrade version: %s", upgradeVersion)
3736

3837
// Create RayService custom resource
39-
rayServiceAC := rayv1ac.RayService(rayServiceName, namespace.Name).WithSpec(e2e.RayServiceSampleYamlApplyConfiguration())
38+
rayServiceAC := rayv1ac.RayService(rayServiceName, namespace.Name).WithSpec(rayServiceSampleYamlApplyConfigurationWithWorker())
4039
rayService, err := test.Client().Ray().RayV1().RayServices(namespace.Name).Apply(test.Ctx(), rayServiceAC, TestApplyOptions)
4140
g.Expect(err).NotTo(HaveOccurred())
4241
g.Expect(rayService).NotTo(BeNil())
@@ -65,11 +64,11 @@ func TestZeroDowntimeUpgradeAfterOperatorUpgrade(t *testing.T) {
6564

6665
// Validate RayService serve service correctly configured
6766
svcName := utils.GenerateServeServiceName(rayService.Name)
68-
test.T().Logf("Checking that the K8s serve service %s has exactly one endpoint because the cluster only has a head Pod", svcName)
67+
test.T().Logf("Checking that the K8s serve service %s has exactly one endpoint and two addresses", svcName)
6968
endpoints, err := test.Client().Core().CoreV1().Endpoints(namespace.Name).Get(test.Ctx(), svcName, metav1.GetOptions{})
7069
g.Expect(err).NotTo(HaveOccurred())
7170
g.Expect(endpoints.Subsets).To(HaveLen(1))
72-
g.Expect(endpoints.Subsets[0].Addresses).To(HaveLen(1))
71+
g.Expect(endpoints.Subsets[0].Addresses).To(HaveLen(2))
7372

7473
// Upgrade KubeRay operator to latest version and replace CRDs
7574
test.T().Logf("Upgrading the KubeRay operator to %s", upgradeVersion)

‎ray-operator/test/e2eupgrade/support.go

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,12 @@ import (
44
"os/exec"
55

66
corev1 "k8s.io/api/core/v1"
7+
"k8s.io/apimachinery/pkg/api/resource"
8+
corev1ac "k8s.io/client-go/applyconfigurations/core/v1"
79

810
rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1"
11+
"github.com/ray-project/kuberay/ray-operator/controllers/ray/utils"
12+
rayv1ac "github.com/ray-project/kuberay/ray-operator/pkg/client/applyconfiguration/ray/v1"
913
e2e "github.com/ray-project/kuberay/ray-operator/test/e2erayservice"
1014
. "github.com/ray-project/kuberay/ray-operator/test/support"
1115
)
@@ -23,3 +27,100 @@ func requestRayService(t Test, rayService *rayv1.RayService, curlPod *corev1.Pod
2327

2428
return stdout1.String() + ", " + stdout2.String()
2529
}
30+
31+
func rayServiceSampleYamlApplyConfigurationWithWorker() *rayv1ac.RayServiceSpecApplyConfiguration {
32+
return rayv1ac.RayServiceSpec().WithServeConfigV2(`applications:
33+
- name: fruit_app
34+
import_path: fruit.deployment_graph
35+
route_prefix: /fruit
36+
runtime_env:
37+
working_dir: "https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip"
38+
deployments:
39+
- name: MangoStand
40+
num_replicas: 1
41+
user_config:
42+
price: 3
43+
ray_actor_options:
44+
num_cpus: 0.1
45+
- name: OrangeStand
46+
num_replicas: 1
47+
user_config:
48+
price: 2
49+
ray_actor_options:
50+
num_cpus: 0.1
51+
- name: FruitMarket
52+
num_replicas: 1
53+
ray_actor_options:
54+
num_cpus: 0.1
55+
- name: math_app
56+
import_path: conditional_dag.serve_dag
57+
route_prefix: /calc
58+
runtime_env:
59+
working_dir: "https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip"
60+
deployments:
61+
- name: Adder
62+
num_replicas: 1
63+
user_config:
64+
increment: 3
65+
ray_actor_options:
66+
num_cpus: 0.1
67+
- name: Multiplier
68+
num_replicas: 1
69+
user_config:
70+
factor: 5
71+
ray_actor_options:
72+
num_cpus: 0.1
73+
- name: Router
74+
ray_actor_options:
75+
num_cpus: 0.1
76+
num_replicas: 1`).
77+
WithRayClusterSpec(rayv1ac.RayClusterSpec().
78+
WithRayVersion(GetRayVersion()).
79+
WithHeadGroupSpec(rayv1ac.HeadGroupSpec().
80+
WithRayStartParams(map[string]string{"dashboard-host": "0.0.0.0"}).
81+
WithTemplate(corev1ac.PodTemplateSpec().
82+
WithSpec(corev1ac.PodSpec().
83+
WithContainers(corev1ac.Container().
84+
WithName("ray-head").
85+
WithImage(GetRayImage()).
86+
WithPorts(
87+
corev1ac.ContainerPort().WithName(utils.GcsServerPortName).WithContainerPort(utils.DefaultGcsServerPort),
88+
corev1ac.ContainerPort().WithName(utils.ServingPortName).WithContainerPort(utils.DefaultServingPort),
89+
corev1ac.ContainerPort().WithName(utils.DashboardPortName).WithContainerPort(utils.DefaultDashboardPort),
90+
corev1ac.ContainerPort().WithName(utils.ClientPortName).WithContainerPort(utils.DefaultClientPort),
91+
).
92+
WithResources(corev1ac.ResourceRequirements().
93+
WithRequests(corev1.ResourceList{
94+
corev1.ResourceCPU: resource.MustParse("1"),
95+
corev1.ResourceMemory: resource.MustParse("2Gi"),
96+
}).
97+
WithLimits(corev1.ResourceList{
98+
corev1.ResourceCPU: resource.MustParse("2"),
99+
corev1.ResourceMemory: resource.MustParse("3Gi"),
100+
})))))).
101+
WithWorkerGroupSpecs(rayv1ac.WorkerGroupSpec().
102+
WithReplicas(1).
103+
WithGroupName("ray-worker-1").
104+
WithMinReplicas(1).
105+
WithMaxReplicas(1).
106+
WithRayStartParams(map[string]string{"num-cpus": "1"}).
107+
WithTemplate(corev1ac.PodTemplateSpec().
108+
WithSpec(corev1ac.PodSpec().
109+
WithContainers(corev1ac.Container().
110+
WithName("ray-worker").
111+
WithImage(GetRayImage()).
112+
WithResources(corev1ac.ResourceRequirements().
113+
WithRequests(corev1.ResourceList{
114+
corev1.ResourceCPU: resource.MustParse("500m"),
115+
corev1.ResourceMemory: resource.MustParse("1Gi"),
116+
}).
117+
WithLimits(corev1.ResourceList{
118+
corev1.ResourceCPU: resource.MustParse("1"),
119+
corev1.ResourceMemory: resource.MustParse("2Gi"),
120+
}),
121+
),
122+
),
123+
),
124+
),
125+
))
126+
}

0 commit comments

Comments
 (0)