Skip to content

Commit f7f0191

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #157 from harshanarayana/feature/GIT-138/enable-mutli-cluster-workflows
multi-cluster workflow: add example of multi cluster test workflow
2 parents 2aa1046 + e1b7605 commit f7f0191

File tree

11 files changed

+308
-0
lines changed

11 files changed

+308
-0
lines changed

‎README.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
# E2E Framework
22

3+
[![godoc](https://pkg.go.dev/badge/github.com/sigs.k8s.io/e2e-framework)](https://pkg.go.dev/sigs.k8s.io/e2e-framework)
4+
35
A Go framework for end-to-end testing of components running in Kubernetes clusters.
46

57
The primary goal of this project is to provide a `go test`(able)

‎examples/multi_cluster/README.md‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
# Multi Cluster Test Runs
2+
3+
This directory contains the example of how to run tests on multiple `kind` clusters at the same time using the same `env.Environment`
4+
5+
## What does this test do ?
6+
7+
1. Create two clusters
8+
1. One with prefix cluster-one
9+
2. One with prefix cluster-two
10+
2. Install A sample helm chart on both clusters
11+
3. Run an assessment to check if the chart has successfully been deployed by checking the pod status
12+
4. Teardown the Test Environments
13+
14+
# Run Tests
15+
16+
These test cases can be executed using the normal `go test` command by passing the right arguments
17+
18+
```bash
19+
go test -v .
20+
```
21+
22+
```bash
23+
go test -c -o multi-cluster.test .
24+
25+
./multi-cluster.test --v 4
26+
```

‎examples/multi_cluster/deployment_test.go‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
/*
2+
Copyright 2022 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package multi_cluster
18+
19+
import (
20+
"context"
21+
"fmt"
22+
"os"
23+
"path/filepath"
24+
"testing"
25+
26+
appsv1 "k8s.io/api/apps/v1"
27+
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
28+
"sigs.k8s.io/e2e-framework/klient"
29+
"sigs.k8s.io/e2e-framework/klient/k8s"
30+
"sigs.k8s.io/e2e-framework/klient/wait"
31+
"sigs.k8s.io/e2e-framework/klient/wait/conditions"
32+
"sigs.k8s.io/e2e-framework/pkg/envconf"
33+
"sigs.k8s.io/e2e-framework/pkg/envfuncs"
34+
"sigs.k8s.io/e2e-framework/pkg/features"
35+
"sigs.k8s.io/e2e-framework/third_party/helm"
36+
)
37+
38+
var curDir, _ = os.Getwd()
39+
40+
func checkPodStatus(t *testing.T, kubeConfig string, clusterName string) {
41+
t.Helper()
42+
client, err := klient.NewWithKubeConfigFile(kubeConfig)
43+
if err != nil {
44+
t.Errorf("ran into an error trying to create a client for Cluster %s", clusterName)
45+
}
46+
deployment := &appsv1.Deployment{
47+
ObjectMeta: v1.ObjectMeta{
48+
Name: "example",
49+
Namespace: "default",
50+
},
51+
Spec: appsv1.DeploymentSpec{},
52+
}
53+
err = wait.For(conditions.New(client.Resources()).ResourceScaled(deployment, func(object k8s.Object) int32 {
54+
return object.(*appsv1.Deployment).Status.ReadyReplicas
55+
}, 1))
56+
if err != nil {
57+
t.Fatal("failed waiting for the Deployment to reach a ready state")
58+
}
59+
}
60+
61+
func TestScenarioOne(t *testing.T) {
62+
feature := features.New("Scenario One").
63+
Setup(func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context {
64+
for _, clusterName := range clusterNames {
65+
cluster, ok := envfuncs.GetKindClusterFromContext(ctx, clusterName)
66+
if !ok {
67+
t.Fatalf("Failed to extract kind cluster %s from context", clusterName)
68+
}
69+
manager := helm.New(cluster.GetKubeconfig())
70+
err := manager.RunInstall(helm.WithName("example"), helm.WithNamespace("default"), helm.WithChart(filepath.Join(curDir, "testdata", "example_chart")), helm.WithWait(), helm.WithTimeout("10m"))
71+
if err != nil {
72+
t.Fatal("failed to invoke helm install operation due to an error", err)
73+
}
74+
}
75+
return ctx
76+
}).
77+
Assess(fmt.Sprintf("Deployment is running successfully - %s", clusterNames[0]), func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context {
78+
cluster, ok := envfuncs.GetKindClusterFromContext(ctx, clusterNames[0])
79+
if !ok {
80+
t.Fatalf("Failed to extract kind cluster %s from context", clusterNames[0])
81+
}
82+
checkPodStatus(t, cluster.GetKubeconfig(), clusterNames[0])
83+
return ctx
84+
}).
85+
Assess(fmt.Sprintf("Deployment is running successfully - %s", clusterNames[1]), func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context {
86+
cluster, ok := envfuncs.GetKindClusterFromContext(ctx, clusterNames[1])
87+
if !ok {
88+
t.Fatalf("Failed to extract kind cluster %s from context", clusterNames[1])
89+
}
90+
checkPodStatus(t, cluster.GetKubeconfig(), clusterNames[1])
91+
return ctx
92+
}).
93+
Feature()
94+
95+
testEnv.Test(t, feature)
96+
}

‎examples/multi_cluster/main_test.go‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
/*
2+
Copyright 2022 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package multi_cluster
18+
19+
import (
20+
"context"
21+
"os"
22+
"testing"
23+
24+
"sigs.k8s.io/e2e-framework/pkg/env"
25+
"sigs.k8s.io/e2e-framework/pkg/envconf"
26+
"sigs.k8s.io/e2e-framework/pkg/envfuncs"
27+
)
28+
29+
var (
30+
testEnv env.Environment
31+
clusterNames []string
32+
)
33+
34+
func TestMain(m *testing.M) {
35+
cfg, _ := envconf.NewFromFlags()
36+
testEnv = env.NewWithConfig(cfg)
37+
38+
clusterNames = []string{
39+
envconf.RandomName("cluster-one", 16),
40+
envconf.RandomName("cluster-two", 16),
41+
}
42+
43+
testEnv.Setup(
44+
func(ctx context.Context, config *envconf.Config) (context.Context, error) {
45+
var err error
46+
for _, cluster := range clusterNames {
47+
ctx, err = envfuncs.CreateKindCluster(cluster)(ctx, config)
48+
if err != nil {
49+
return ctx, err
50+
}
51+
}
52+
return ctx, nil
53+
},
54+
).Finish(
55+
func(ctx context.Context, config *envconf.Config) (context.Context, error) {
56+
var err error
57+
for _, cluster := range clusterNames {
58+
ctx, err = envfuncs.DestroyKindCluster(cluster)(ctx, config)
59+
if err != nil {
60+
return ctx, err
61+
}
62+
}
63+
return ctx, nil
64+
},
65+
)
66+
67+
os.Exit(testEnv.Run(m))
68+
}

‎examples/multi_cluster/testdata/example_chart/.helmignore‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
# Patterns to ignore when building packages.
2+
# This supports shell glob matching, relative path matching, and
3+
# negation (prefixed with !). Only one pattern per line.
4+
.DS_Store
5+
# Common VCS dirs
6+
.git/
7+
.gitignore
8+
.bzr/
9+
.bzrignore
10+
.hg/
11+
.hgignore
12+
.svn/
13+
# Common backup files
14+
*.swp
15+
*.bak
16+
*.tmp
17+
*.orig
18+
*~
19+
# Various IDEs
20+
.project
21+
.idea/
22+
*.tmproj
23+
.vscode/

‎examples/multi_cluster/testdata/example_chart/Chart.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
apiVersion: v2
2+
name: example
3+
description: An example Helm chart for Kubernetes e2e-framework
4+
type: application
5+
version: 0.1.0
6+
appVersion: "1.16.0"

‎examples/multi_cluster/testdata/example_chart/templates/deployment.yaml‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
apiVersion: apps/v1
2+
kind: Deployment
3+
metadata:
4+
name: {{ .Release.Name }}
5+
namespace: {{ .Release.Namespace }}
6+
labels:
7+
app: {{ .Release.Name }}
8+
spec:
9+
replicas: {{ .Values.replicaCount }}
10+
selector:
11+
matchLabels:
12+
app: {{ .Release.Name }}
13+
template:
14+
metadata:
15+
labels:
16+
app: {{ .Release.Name }}
17+
spec:
18+
automountServiceAccountToken: true
19+
containers:
20+
- name: {{ .Chart.Name }}
21+
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
22+
imagePullPolicy: {{ .Values.image.pullPolicy }}
23+
ports:
24+
- name: http
25+
containerPort: {{ .Values.service.port }}
26+
protocol: TCP
27+
livenessProbe:
28+
httpGet:
29+
path: /
30+
port: http
31+
readinessProbe:
32+
httpGet:
33+
path: /
34+
port: http
35+

‎examples/multi_cluster/testdata/example_chart/templates/service.yaml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
apiVersion: v1
2+
kind: Service
3+
metadata:
4+
name: {{ .Release.Name }}
5+
namespace: {{ .Release.Namespace }}
6+
labels:
7+
app: {{ .Release.Name }}
8+
spec:
9+
type: {{ .Values.service.type }}
10+
ports:
11+
- port: {{ .Values.service.port }}
12+
targetPort: http
13+
protocol: TCP
14+
name: http
15+
selector:
16+
app: {{ .Release.Name }}

‎examples/multi_cluster/testdata/example_chart/templates/tests/test-connection.yaml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
apiVersion: v1
2+
kind: Pod
3+
metadata:
4+
name: "{{ .Release.Name }}-test-connection"
5+
namespace: {{ .Release.Namespace }}
6+
annotations:
7+
"helm.sh/hook": test
8+
spec:
9+
containers:
10+
- name: wget
11+
image: busybox
12+
command: ['wget']
13+
args: ['{{ .Release.Name }}:{{ .Values.service.port }}']
14+
restartPolicy: Never

‎examples/multi_cluster/testdata/example_chart/values.yaml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
replicaCount: 1
2+
3+
4+
image:
5+
repository: nginx
6+
pullPolicy: IfNotPresent
7+
tag: ""
8+
9+
service:
10+
type: ClusterIP
11+
port: 80

0 commit comments

Comments
 (0)