Skip to content

Commit f8b0bcd

Browse files
harshanarayanaharshanarayana
committed
provider: added k3d provider and lifecycle handlers
This change includes the following changes and features. 1. Added a new Interface type `E2EClusterProviderWithLifeCycle` which can be used to setup providers that extend the cluster lifecycle function around the nodes. 2. Enabled a `k3d` based provider with support for Node lifecycle management. 3. Existing Image loader related function and interfaces were augmented with `args ...string` to be able to provide additional arguments in case if the image load handlers need some of the additional config. refactor: re-organize the provider under third_party uptick kind version to 0.26.0 and fix vcluster test
1 parent fc5ec37 commit f8b0bcd

File tree

17 files changed

+1517
-654
lines changed

17 files changed

+1517
-654
lines changed

‎examples/k3d/k3d_test.go

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
/*
2+
Copyright 2024 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package k3d
18+
19+
import (
20+
"context"
21+
"fmt"
22+
"testing"
23+
"time"
24+
25+
"sigs.k8s.io/e2e-framework/pkg/stepfuncs"
26+
"sigs.k8s.io/e2e-framework/support"
27+
28+
appsv1 "k8s.io/api/apps/v1"
29+
corev1 "k8s.io/api/core/v1"
30+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
31+
"sigs.k8s.io/e2e-framework/klient/wait"
32+
"sigs.k8s.io/e2e-framework/klient/wait/conditions"
33+
"sigs.k8s.io/e2e-framework/pkg/envconf"
34+
"sigs.k8s.io/e2e-framework/pkg/features"
35+
)
36+
37+
func newDeployment(namespace string, name string, replicaCount int32) *appsv1.Deployment {
38+
podSpec := corev1.PodSpec{
39+
Containers: []corev1.Container{
40+
{
41+
Name: "my-container",
42+
Image: "nginx",
43+
},
44+
},
45+
}
46+
return &appsv1.Deployment{
47+
ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace, Labels: map[string]string{"app": "test-app"}},
48+
Spec: appsv1.DeploymentSpec{
49+
Replicas: &replicaCount,
50+
Selector: &metav1.LabelSelector{
51+
MatchLabels: map[string]string{"app": "test-app"},
52+
},
53+
Template: corev1.PodTemplateSpec{
54+
ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"app": "test-app"}},
55+
Spec: podSpec,
56+
},
57+
},
58+
}
59+
}
60+
61+
func TestK3DCluster(t *testing.T) {
62+
deploymentFeature := features.New("Should be able to create a new deployment in the k3d cluster").
63+
Assess("Create a new deployment", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context {
64+
deployment := newDeployment(c.Namespace(), "test-deployment", 1)
65+
if err := c.Client().Resources().Create(ctx, deployment); err != nil {
66+
t.Fatal(err)
67+
}
68+
var dep appsv1.Deployment
69+
if err := c.Client().Resources().Get(ctx, "test-deployment", c.Namespace(), &dep); err != nil {
70+
t.Fatal(err)
71+
}
72+
err := wait.For(conditions.New(c.Client().Resources()).DeploymentConditionMatch(&dep, appsv1.DeploymentAvailable, corev1.ConditionTrue), wait.WithTimeout(time.Minute*3))
73+
if err != nil {
74+
t.Fatal(err)
75+
}
76+
return context.WithValue(ctx, "test-deployment", &dep)
77+
}).
78+
Feature()
79+
80+
nodeAddFeature := features.New("Should be able to add a new node to the k3d cluster").
81+
Setup(stepfuncs.PerformNodeOperation(support.AddNode, &support.Node{
82+
Name: fmt.Sprintf("%s-agent", clusterName),
83+
Cluster: clusterName,
84+
Role: "agent",
85+
})).
86+
Assess("Check if the node is added to the cluster", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context {
87+
var node corev1.Node
88+
if err := c.Client().Resources().Get(ctx, fmt.Sprintf("k3d-%s-agent-0", clusterName), c.Namespace(), &node); err != nil {
89+
t.Fatal(err)
90+
}
91+
return ctx
92+
}).Feature()
93+
94+
testEnv.Test(t, deploymentFeature, nodeAddFeature)
95+
}

‎examples/k3d/main_test.go

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
/*
2+
Copyright 2024 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package k3d
18+
19+
import (
20+
"os"
21+
"testing"
22+
23+
"sigs.k8s.io/e2e-framework/pkg/env"
24+
"sigs.k8s.io/e2e-framework/pkg/envconf"
25+
"sigs.k8s.io/e2e-framework/pkg/envfuncs"
26+
"sigs.k8s.io/e2e-framework/support/k3d"
27+
)
28+
29+
var (
30+
testEnv env.Environment
31+
clusterName string
32+
)
33+
34+
func TestMain(m *testing.M) {
35+
testEnv = env.New()
36+
clusterName = envconf.RandomName("test", 16)
37+
namespace := envconf.RandomName("k3d-ns", 16)
38+
39+
testEnv.Setup(
40+
envfuncs.CreateClusterWithOpts(k3d.NewProvider(), clusterName, k3d.WithImage("rancher/k3s:v1.29.6-k3s1")),
41+
envfuncs.CreateNamespace(namespace),
42+
envfuncs.LoadImageToCluster(clusterName, "rancher/k3s:v1.29.6-k3s1", "--verbose", "--mode", "direct"),
43+
)
44+
45+
testEnv.Finish(
46+
envfuncs.DeleteNamespace(namespace),
47+
envfuncs.DestroyCluster(clusterName),
48+
)
49+
50+
os.Exit(testEnv.Run(m))
51+
}

‎pkg/envfuncs/provider_funcs.go

Lines changed: 34 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -20,19 +20,19 @@ import (
2020
"context"
2121
"fmt"
2222

23+
"sigs.k8s.io/e2e-framework/pkg/utils"
24+
2325
"sigs.k8s.io/e2e-framework/pkg/env"
2426
"sigs.k8s.io/e2e-framework/pkg/envconf"
2527
"sigs.k8s.io/e2e-framework/support"
2628
)
2729

28-
type clusterNameContextKey string
29-
3030
var LoadDockerImageToCluster = LoadImageToCluster
3131

3232
// GetClusterFromContext helps extract the E2EClusterProvider object from the context.
3333
// This can be used to setup and run tests of multi cluster e2e Prioviders.
3434
func GetClusterFromContext(ctx context.Context, clusterName string) (support.E2EClusterProvider, bool) {
35-
c := ctx.Value(clusterNameContextKey(clusterName))
35+
c := ctx.Value(support.ClusterNameContextKey(clusterName))
3636
if c == nil {
3737
return nil, false
3838
}
@@ -47,8 +47,19 @@ func GetClusterFromContext(ctx context.Context, clusterName string) (support.E2E
4747
// NOTE: the returned function will update its env config with the
4848
// kubeconfig file for the config client.
4949
func CreateCluster(p support.E2EClusterProvider, clusterName string) env.Func {
50+
return CreateClusterWithOpts(p, clusterName)
51+
}
52+
53+
// CreateClusterWithOpts returns an env.Func that is used to
54+
// create an E2E provider cluster that is then injected in the context
55+
// using the name as a key. This can be provided with additional opts to extend the create
56+
// workflow of the cluster.
57+
//
58+
// NOTE: the returned function will update its env config with the
59+
// kubeconfig file for the config client.
60+
func CreateClusterWithOpts(p support.E2EClusterProvider, clusterName string, opts ...support.ClusterOpts) env.Func {
5061
return func(ctx context.Context, cfg *envconf.Config) (context.Context, error) {
51-
k := p.SetDefaults().WithName(clusterName)
62+
k := p.SetDefaults().WithName(clusterName).WithOpts(opts...)
5263
kubecfg, err := k.Create(ctx)
5364
if err != nil {
5465
return ctx, err
@@ -63,7 +74,7 @@ func CreateCluster(p support.E2EClusterProvider, clusterName string) env.Func {
6374
}
6475

6576
// store entire cluster value in ctx for future access using the cluster name
66-
return context.WithValue(ctx, clusterNameContextKey(clusterName), k), nil
77+
return context.WithValue(ctx, support.ClusterNameContextKey(clusterName), k), nil
6778
}
6879
}
6980

@@ -90,7 +101,7 @@ func CreateClusterWithConfig(p support.E2EClusterProvider, clusterName, configFi
90101
}
91102

92103
// store entire cluster value in ctx for future access using the cluster name
93-
return context.WithValue(ctx, clusterNameContextKey(clusterName), k), nil
104+
return context.WithValue(ctx, support.ClusterNameContextKey(clusterName), k), nil
94105
}
95106
}
96107

@@ -100,7 +111,7 @@ func CreateClusterWithConfig(p support.E2EClusterProvider, clusterName, configFi
100111
// NOTE: this should be used in a Environment.Finish step.
101112
func DestroyCluster(name string) env.Func {
102113
return func(ctx context.Context, cfg *envconf.Config) (context.Context, error) {
103-
clusterVal := ctx.Value(clusterNameContextKey(name))
114+
clusterVal := ctx.Value(support.ClusterNameContextKey(name))
104115
if clusterVal == nil {
105116
return ctx, fmt.Errorf("destroy e2e provider cluster func: context cluster is nil")
106117
}
@@ -121,9 +132,9 @@ func DestroyCluster(name string) env.Func {
121132
// LoadImageToCluster returns an EnvFunc that
122133
// retrieves a previously saved e2e provider Cluster in the context (using the name), and then loads a container image
123134
// from the host into the cluster.
124-
func LoadImageToCluster(name, image string) env.Func {
135+
func LoadImageToCluster(name, image string, args ...string) env.Func {
125136
return func(ctx context.Context, cfg *envconf.Config) (context.Context, error) {
126-
clusterVal := ctx.Value(clusterNameContextKey(name))
137+
clusterVal := ctx.Value(support.ClusterNameContextKey(name))
127138
if clusterVal == nil {
128139
return ctx, fmt.Errorf("load image func: context cluster is nil")
129140
}
@@ -133,7 +144,7 @@ func LoadImageToCluster(name, image string) env.Func {
133144
return ctx, fmt.Errorf("load image archive func: cluster provider does not support LoadImage helper")
134145
}
135146

136-
if err := cluster.LoadImage(ctx, image); err != nil {
147+
if err := cluster.LoadImage(ctx, image, args...); err != nil {
137148
return ctx, fmt.Errorf("load image: %w", err)
138149
}
139150

@@ -144,9 +155,9 @@ func LoadImageToCluster(name, image string) env.Func {
144155
// LoadImageArchiveToCluster returns an EnvFunc that
145156
// retrieves a previously saved e2e provider Cluster in the context (using the name), and then loads a container image TAR archive
146157
// from the host into the cluster.
147-
func LoadImageArchiveToCluster(name, imageArchive string) env.Func {
158+
func LoadImageArchiveToCluster(name, imageArchive string, args ...string) env.Func {
148159
return func(ctx context.Context, cfg *envconf.Config) (context.Context, error) {
149-
clusterVal := ctx.Value(clusterNameContextKey(name))
160+
clusterVal := ctx.Value(support.ClusterNameContextKey(name))
150161
if clusterVal == nil {
151162
return ctx, fmt.Errorf("load image archive func: context cluster is nil")
152163
}
@@ -156,7 +167,7 @@ func LoadImageArchiveToCluster(name, imageArchive string) env.Func {
156167
return ctx, fmt.Errorf("load image archive func: cluster provider does not support LoadImageArchive helper")
157168
}
158169

159-
if err := cluster.LoadImageArchive(ctx, imageArchive); err != nil {
170+
if err := cluster.LoadImageArchive(ctx, imageArchive, args...); err != nil {
160171
return ctx, fmt.Errorf("load image archive: %w", err)
161172
}
162173

@@ -169,7 +180,7 @@ func LoadImageArchiveToCluster(name, imageArchive string) env.Func {
169180
// in the provided destination.
170181
func ExportClusterLogs(name, dest string) env.Func {
171182
return func(ctx context.Context, cfg *envconf.Config) (context.Context, error) {
172-
clusterVal := ctx.Value(clusterNameContextKey(name))
183+
clusterVal := ctx.Value(support.ClusterNameContextKey(name))
173184
if clusterVal == nil {
174185
return ctx, fmt.Errorf("export e2e provider cluster logs: context cluster is nil")
175186
}
@@ -186,3 +197,12 @@ func ExportClusterLogs(name, dest string) env.Func {
186197
return ctx, nil
187198
}
188199
}
200+
201+
// PerformNodeOperation returns an EnvFunc that can be used to perform some node lifecycle operations.
202+
// This can be used to add/remove/start/stop nodes in the cluster.
203+
func PerformNodeOperation(clusterName string, action support.NodeOperation, node *support.Node, args ...string) env.Func {
204+
return func(ctx context.Context, cfg *envconf.Config) (context.Context, error) {
205+
err := utils.PerformNodeLifecycleOperation(ctx, action, node, args...)
206+
return ctx, err
207+
}
208+
}

‎pkg/stepfuncs/nodelifecycle_funcs.go

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
/*
2+
Copyright 2024 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package stepfuncs
18+
19+
import (
20+
"context"
21+
"testing"
22+
23+
"sigs.k8s.io/e2e-framework/pkg/utils"
24+
25+
"sigs.k8s.io/e2e-framework/pkg/envconf"
26+
"sigs.k8s.io/e2e-framework/pkg/types"
27+
"sigs.k8s.io/e2e-framework/support"
28+
)
29+
30+
// PerformNodeOperation returns a step function that performs a node operation on a cluster.
31+
// This can be integrated as a setup function for a feature in question before the feature
32+
// is tested.
33+
func PerformNodeOperation(action support.NodeOperation, node *support.Node, args ...string) types.StepFunc {
34+
return func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context {
35+
t.Helper()
36+
37+
err := utils.PerformNodeLifecycleOperation(ctx, action, node, args...)
38+
if err != nil {
39+
t.Fatalf("failed to perform node operation: %v", err)
40+
}
41+
return ctx
42+
}
43+
}

0 commit comments

Comments
 (0)