Skip to content

Commit 62ad934

Browse files
machichimamachichima
authored
Refactor Apiserver e2e run in cluster (#3529)
* feat: add apiserver exec client for CreateComputeTemplate directly execute command in the pod instead of through http -e Signed-off-by: machichima <[email protected]> * build: install curl in apiserver container -e Signed-off-by: machichima <[email protected]> * feat: exec Delete Compute Template without http -e Signed-off-by: machichima <[email protected]> * feat: add exec.go for sending commands without http in e2e test with a kubectl exec like behaviour -e Signed-off-by: machichima <[email protected]> * feat: run curl container in pod alongside apiserver -e Signed-off-by: machichima <[email protected]> * feat: extract http status from curl output -e Signed-off-by: machichima <[email protected]> * style: remove unused comment -e Signed-off-by: machichima <[email protected]> * fix: wrong image name -e Signed-off-by: machichima <[email protected]> * style: remove redundant comment -e Signed-off-by: machichima <[email protected]> * build: NodePort to ClusterIP in deploy base insecure -e Signed-off-by: machichima <[email protected]> * build: update buildkit test-e2e to setup apiserver e2e -e Signed-off-by: machichima <[email protected]> * fix: alter executeRequest only -e Signed-off-by: machichima <[email protected]> * feat: remove extraPort and NodePort for apiserver e2e test Signed-off-by: machichima <[email protected]> * docs: add comment on e2e kustomization.yaml Signed-off-by: machichima <[email protected]> * fix: remove unused Docker & Make command Signed-off-by: machichima <[email protected]> * fix: more comment description + function to private Signed-off-by: machichima <[email protected]> * refactor: more intuitive make command Signed-off-by: machichima <[email protected]> * docs: make comment more concise Signed-off-by: machichima <[email protected]> * fix: use config.GetConfig() Signed-off-by: machichima <[email protected]> * fix: update make command Signed-off-by: machichima <[email protected]> * fix: update buildkite apiserver make cmd Signed-off-by: machichima <[email protected]> * feat: enable kubectl execute for apiserversdk e2e test Signed-off-by: machichima <[email protected]> * fix: apiserversdk e2e use k8s service proxy Signed-off-by: machichima <[email protected]> * docs: add docstring Signed-off-by: machichima <[email protected]> * refactor: remove adding suffix / Signed-off-by: machichima <[email protected]> * nit: wrap some words for 3rd party errors Signed-off-by: machichima <[email protected]> * nit: TODO for migrate SPDY to websocket Signed-off-by: machichima <[email protected]> --------- Signed-off-by: machichima <[email protected]> Signed-off-by: Nary Yeh <[email protected]>
1 parent d0b6337 commit 62ad934

File tree

11 files changed

+336
-22
lines changed

11 files changed

+336
-22
lines changed

‎.buildkite/test-e2e.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -110,12 +110,12 @@
110110
- kubectl wait --timeout=90s --for=condition=Available=true deployment kuberay-operator
111111
# Build and start apiserver
112112
- pushd ../apiserver
113-
- KIND_CLUSTER_NAME=kind KIND=kind make install
113+
- KIND_CLUSTER_NAME=kind KIND=kind make install-apiserver-e2e
114114
- kubectl wait --namespace ray-system --for=condition=Available --timeout=90s deployment/kuberay-apiserver -n ray-system
115115
# Run e2e tests and print KubeRay api server logs if tests fail
116116
- echo "--- START:Running e2e apiserver (nightly operator) tests"
117117
- set -o pipefail
118118
- mkdir -p "$(pwd)/tmp" && export KUBERAY_TEST_OUTPUT_DIR=$(pwd)/tmp
119119
- echo "KUBERAY_TEST_OUTPUT_DIR=$$KUBERAY_TEST_OUTPUT_DIR"
120-
- E2E_API_SERVER_URL="http://docker:31888" go test -parallel 4 -timeout 60m -v ./test/e2e/... 2>&1 | awk -f ../.buildkite/format.awk | tee $$KUBERAY_TEST_OUTPUT_DIR/gotest.log || (kubectl logs -l app.kubernetes.io/component=kuberay-apiserver --namespace ray-system | tee $$KUBERAY_TEST_OUTPUT_DIR/kuberay-apiserver.log && cd $$KUBERAY_TEST_OUTPUT_DIR && find . -name "*.log" | tar -cf /artifact-mount/e2e-apiserver-log.tar -T - && exit 1)
120+
- E2E_API_SERVER_URL="http://localhost:8888" go test -parallel 4 -timeout 60m -v ./test/e2e/... 2>&1 | awk -f ../.buildkite/format.awk | tee $$KUBERAY_TEST_OUTPUT_DIR/gotest.log || (kubectl logs -l app.kubernetes.io/component=kuberay-apiserver --namespace ray-system | tee $$KUBERAY_TEST_OUTPUT_DIR/kuberay-apiserver.log && cd $$KUBERAY_TEST_OUTPUT_DIR && find . -name "*.log" | tar -cf /artifact-mount/e2e-apiserver-log.tar -T - && exit 1)
121121
- echo "--- END:Apiserver e2e (nightly operator) tests finished"

‎apiserver/Makefile

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,10 @@ help: ## Display this help.
5959
.PHONY: start-local-apiserver
6060
start-local-apiserver: operator-image cluster load-operator-image deploy-operator install ## Build and start apiserver from scratch.
6161

62+
# Build and start apiserver and curl from scratch.
63+
.PHONY: start-local-apiserver-e2e
64+
start-local-apiserver-e2e: operator-image cluster load-operator-image deploy-operator install-apiserver-e2e
65+
6266
.PHONY: clear-local-apiserver
6367
clear-local-apiserver: clean-cluster ## Clear local apiserver.
6468

@@ -116,7 +120,7 @@ e2e-test: ## Run end to end tests using a pre-existing cluster.
116120
# 4. Run the end to end tests.
117121
# 5. Clean up the cluster.
118122
.PHONY: local-e2e-test
119-
local-e2e-test: clean-cluster start-local-apiserver load-ray-test-image e2e-test clean-cluster
123+
local-e2e-test: clean-cluster start-local-apiserver-e2e load-ray-test-image e2e-test clean-cluster
120124

121125
##@ Testing Setup
122126
KIND_CONFIG ?= hack/kind-cluster-config.yaml
@@ -190,10 +194,20 @@ install: kustomize docker-image load-image ## Install the kuberay api server wi
190194
cd deploy/local/insecure && $(KUSTOMIZE) edit set image kuberay/apiserver=$(IMG)
191195
$(KUSTOMIZE) build deploy/local/insecure | kubectl create -f -
192196

197+
# Install apiserver and curl for e2e test
198+
.PHONY: install-apiserver-e2e
199+
install-apiserver-e2e: kustomize docker-image load-image ## Install the kuberay api server without security to the K8s cluster specified in ~/.kube/config.
200+
cd deploy/local/e2e && $(KUSTOMIZE) edit set image kuberay/apiserver=$(IMG)
201+
$(KUSTOMIZE) build deploy/local/e2e | kubectl create -f -
202+
193203
.PHONY: uninstall
194204
uninstall: ## Remove the kuberay api server without security server from the K8s cluster specified in ~/.kube/config.
195205
$(KUSTOMIZE) build deploy/local/insecure | kubectl delete -f -
196206

207+
.PHONY: uninstall-apiserver-e2e
208+
uninstall-apiserver-e2e: ## Remove the kuberay api server for e2e from the K8s cluster specified in ~/.kube/config.
209+
$(KUSTOMIZE) build deploy/local/e2e | kubectl delete -f -
210+
197211
.PHONY: install-secure
198212
install-secure: kustomize docker-image security-proxy-image load-image load-security-proxy-image ## Install the kuberay api server with security to the K8s cluster specified in ~/.kube/config.
199213
cd deploy/local/secure && $(KUSTOMIZE) edit set image kuberay/apiserver=$(IMG) && $(KUSTOMIZE) edit set image kuberay/security-proxy=kuberay/security-proxy:latest

‎apiserver/deploy/local/e2e/kustomization.yaml

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
apiVersion: kustomize.config.k8s.io/v1beta1
2+
kind: Kustomization
3+
resources:
4+
- ../../base/insecure
5+
namespace: ray-system
6+
images:
7+
- name: kuberay/apiserver
8+
newName: quay.io/kuberay/apiserver
9+
newTag: latest
10+
patches:
11+
- patch: |-
12+
- op: replace
13+
path: /spec/template/spec/containers/0/imagePullPolicy
14+
value: IfNotPresent
15+
target:
16+
kind: Deployment
17+
name: kuberay-apiserver
18+
version: v1
19+
- patch: |-
20+
- op: add
21+
path: /spec/template/spec/containers/1
22+
value:
23+
name: apiserver-curl
24+
image: curlimages/curl:latest
25+
command: ["sleep", "infinity"]
26+
imagePullPolicy: IfNotPresent
27+
target:
28+
kind: Deployment
29+
name: kuberay-apiserver
30+
version: v1
31+
# Replace NodePort with ClusterIP as we do not need to receive requests from outside the Kubernetes cluster
32+
- patch: |-
33+
- op: replace
34+
path: /spec/type
35+
value: ClusterIP
36+
- op: remove
37+
path: /spec/ports/0/nodePort
38+
- op: remove
39+
path: /spec/ports/1/nodePort
40+
target:
41+
kind: Service
42+
name: kuberay-apiserver
43+
version: v1

‎apiserver/pkg/http/client.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,11 @@ func NewKuberayAPIServerClient(baseURL string, httpClient *http.Client, retryCfg
8181
return client
8282
}
8383

84+
// Setter function for setting executeHttpRequest method
85+
func (krc *KuberayAPIServerClient) SetExecuteHttpRequest(fn func(httpRequest *http.Request, URL string) ([]byte, *rpcStatus.Status, error)) {
86+
krc.executeHttpRequest = fn
87+
}
88+
8489
// CreateComputeTemplate creates a new compute template.
8590
func (krc *KuberayAPIServerClient) CreateComputeTemplate(request *api.CreateComputeTemplateRequest) (*api.ComputeTemplate, *rpcStatus.Status, error) {
8691
createURL := krc.baseURL + "/apis/v1/namespaces/" + request.Namespace + "/compute_templates"

‎apiserver/pkg/util/config.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ const (
1717
RayClusterEnvironmentLabelKey = "ray.io/environment"
1818
KubernetesApplicationNameLabelKey = "app.kubernetes.io/name"
1919
KubernetesManagedByLabelKey = "app.kubernetes.io/managed-by"
20+
KubernetesComponentLabelKey = "app.kubernetes.io/component"
2021

2122
// Annotation keys
2223
// Role level
@@ -44,6 +45,9 @@ const (
4445
// The component name for apiserver
4546
ComponentName = "kuberay-apiserver"
4647

48+
// The curl container name for apiserver
49+
CurlContainerName = "apiserver-curl"
50+
4751
// Timeout for apiserver gRPC server
4852
GRPCServerDefaultTimeout = 60 * time.Second
4953
)

‎apiserver/test/e2e/apiserversdk/exec.go

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
package apiserversdk
2+
3+
import (
4+
"fmt"
5+
"net/http"
6+
"path"
7+
8+
"k8s.io/client-go/rest"
9+
)
10+
11+
type ProxyRoundTripper struct {
12+
Transport http.RoundTripper
13+
}
14+
15+
func newProxyRoundTripper(cfg *rest.Config) (*ProxyRoundTripper, error) {
16+
transport, err := rest.TransportFor(cfg)
17+
if err != nil {
18+
return nil, fmt.Errorf("failed to create http RoundTripper: %w", err)
19+
}
20+
21+
return &ProxyRoundTripper{
22+
Transport: transport,
23+
}, nil
24+
}
25+
26+
// RoundTrp send the request through the Kubernetes service proxy subresource
27+
func (rt *ProxyRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
28+
newReq := req.Clone(req.Context())
29+
30+
newReq.URL.Path = path.Join(
31+
"/api/v1/namespaces/ray-system/services/kuberay-apiserver:8888/proxy",
32+
req.URL.Path,
33+
)
34+
35+
return rt.Transport.RoundTrip(newReq)
36+
}

‎apiserver/test/e2e/apiserversdk/types.go

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package apiserversdk
22

33
import (
44
"context"
5+
"net/http"
56
"os"
67
"runtime"
78
"strings"
@@ -13,7 +14,6 @@ import (
1314
corev1 "k8s.io/api/core/v1"
1415
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1516
"k8s.io/client-go/kubernetes"
16-
"k8s.io/client-go/rest"
1717
"sigs.k8s.io/controller-runtime/pkg/client/config"
1818

1919
rayv1api "github.com/ray-project/kuberay/ray-operator/apis/ray/v1"
@@ -74,9 +74,15 @@ func newEnd2EndTestingContext(t *testing.T, options ...contextOption) (*End2EndT
7474
}
7575

7676
func withRayHttpClient() contextOption {
77-
return func(_ *testing.T, testingContext *End2EndTestingContext) error {
78-
var err error
79-
testingContext.rayHttpClient, err = rayv1.NewForConfig(&rest.Config{Host: testingContext.apiServerBaseURL})
77+
return func(t *testing.T, testingContext *End2EndTestingContext) error {
78+
kubernetesConfig, err := config.GetConfig()
79+
require.NoError(t, err)
80+
81+
rt, err := newProxyRoundTripper(kubernetesConfig)
82+
require.NoError(t, err)
83+
httpClient := &http.Client{Transport: rt}
84+
85+
testingContext.rayHttpClient, err = rayv1.NewForConfigAndClient(kubernetesConfig, httpClient)
8086
if err != nil {
8187
return err
8288
}
@@ -85,9 +91,11 @@ func withRayHttpClient() contextOption {
8591
}
8692

8793
func withK8sHttpClient() contextOption {
88-
return func(_ *testing.T, testingContext *End2EndTestingContext) error {
89-
var err error
90-
testingContext.k8sHttpClient, err = kubernetes.NewForConfig(&rest.Config{Host: testingContext.apiServerBaseURL})
94+
return func(t *testing.T, testingContext *End2EndTestingContext) error {
95+
kubernetesConfig, err := config.GetConfig()
96+
require.NoError(t, err)
97+
98+
testingContext.k8sHttpClient, err = kubernetes.NewForConfig(kubernetesConfig)
9199
if err != nil {
92100
return err
93101
}
@@ -106,7 +114,7 @@ func withBaseURL() contextOption {
106114
return func(_ *testing.T, testingContext *End2EndTestingContext) error {
107115
baseURL := os.Getenv("E2E_API_SERVER_URL")
108116
if strings.TrimSpace(baseURL) == "" {
109-
baseURL = "http://localhost:31888"
117+
baseURL = "http://localhost:8888"
110118
}
111119
testingContext.apiServerBaseURL = baseURL
112120
return nil

‎apiserver/test/e2e/doc.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
//
99
// - types.go -- provides for data types
1010
// - utils.go -- provides for utility functions
11+
// - exec.go -- provides functions for exec command with kubectl exec
1112
// - cluster_server_e2e_test.go -- provides the test function for the Cluster GRPC Server
1213
// - config_server_e2e_test.go -- provides the test function for the Config GRPC Server
1314
// - job_server_e2e_test.go -- provides the test function for the Job GRPC Server

0 commit comments

Comments
 (0)