Skip to content

Commit 1e36b6f

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request kubernetes#75640 from fabriziopandini/e2e-kubeadm-refactor
e2e kubeadm refactor
2 parents e7eb742 + cb306ae commit 1e36b6f

File tree

11 files changed

+567
-158
lines changed

11 files changed

+567
-158
lines changed

‎test/e2e_kubeadm/BUILD‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,24 +9,28 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library")
99
go_test(
1010
name = "go_default_test",
1111
srcs = [
12+
"bootstrap_token_test.go",
13+
"cluster_info_test.go",
14+
"controlplane_nodes_test.go",
1215
"e2e_kubeadm_suite_test.go",
13-
"kubeadm_test.go",
16+
"kubeadm_config_test.go",
1417
],
1518
out = "e2e_kubeadm.test",
1619
embed = [":go_default_library"],
1720
tags = ["e2e"],
1821
deps = [
22+
"//staging/src/k8s.io/api/authorization/v1:go_default_library",
1923
"//staging/src/k8s.io/api/core/v1:go_default_library",
2024
"//staging/src/k8s.io/api/rbac/v1:go_default_library",
2125
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
2226
"//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library",
2327
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
24-
"//staging/src/k8s.io/client-go/rest:go_default_library",
2528
"//staging/src/k8s.io/cluster-bootstrap/token/api:go_default_library",
2629
"//test/e2e/framework:go_default_library",
2730
"//vendor/github.com/onsi/ginkgo:go_default_library",
2831
"//vendor/github.com/onsi/gomega:go_default_library",
2932
"//vendor/github.com/spf13/pflag:go_default_library",
33+
"//vendor/gopkg.in/yaml.v2:go_default_library",
3034
],
3135
)
3236

@@ -50,11 +54,22 @@ filegroup(
5054

5155
go_library(
5256
name = "go_default_library",
53-
srcs = ["matchers.go"],
57+
srcs = [
58+
"bootstrap_signer.go",
59+
"const.go",
60+
"framework.go",
61+
"util.go",
62+
],
5463
importpath = "k8s.io/kubernetes/test/e2e_kubeadm",
5564
visibility = ["//visibility:public"],
5665
deps = [
66+
"//staging/src/k8s.io/api/authorization/v1:go_default_library",
5767
"//staging/src/k8s.io/api/core/v1:go_default_library",
68+
"//staging/src/k8s.io/api/rbac/v1:go_default_library",
69+
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
70+
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
71+
"//test/e2e/framework:go_default_library",
72+
"//vendor/github.com/onsi/ginkgo:go_default_library",
5873
"//vendor/github.com/onsi/gomega:go_default_library",
5974
"//vendor/github.com/onsi/gomega/gstruct:go_default_library",
6075
],

‎test/e2e_kubeadm/bootstrap_signer.go‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
/*
2+
Copyright 2019 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package e2e_kubeadm
18+
19+
import (
20+
"k8s.io/kubernetes/test/e2e/framework"
21+
22+
. "github.com/onsi/ginkgo"
23+
)
24+
25+
const (
26+
bootstrapTokensSignerRoleName = "system:controller:bootstrap-signer"
27+
)
28+
29+
// Define container for all the test specification aimed at verifying
30+
// that kubeadm creates the bootstrap signer
31+
var _ = KubeadmDescribe("bootstrap signer", func() {
32+
33+
// Get an instance of the k8s test framework
34+
f := framework.NewDefaultFramework("bootstrap token")
35+
36+
// Tests in this container are not expected to create new objects in the cluster
37+
// so we are disabling the creation of a namespace in order to get a faster execution
38+
f.SkipNamespaceCreation = true
39+
40+
It("should be active", func() {
41+
//NB. this is technically implemented a part of the control-plane phase
42+
// and more specifically if the controller manager is properly configured,
43+
// the bootstrapsigner controller is activated and the system:controller:bootstrap-signer
44+
// group will be automatically created
45+
ExpectRole(f.ClientSet, kubeSystemNamespace, bootstrapTokensSignerRoleName)
46+
})
47+
})

‎test/e2e_kubeadm/bootstrap_token_test.go‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
/*
2+
Copyright 2019 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package e2e_kubeadm
18+
19+
import (
20+
corev1 "k8s.io/api/core/v1"
21+
rbacv1 "k8s.io/api/rbac/v1"
22+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
23+
"k8s.io/kubernetes/test/e2e/framework"
24+
25+
. "github.com/onsi/ginkgo"
26+
. "github.com/onsi/gomega"
27+
)
28+
29+
const (
30+
bootstrapTokensGroup = "system:bootstrappers:kubeadm:default-node-token"
31+
bootstrapTokensAllowPostCSRClusterRoleBinding = "kubeadm:kubelet-bootstrap"
32+
bootstrapTokensAllowPostCSRClusterRoleName = "system:node-bootstrapper"
33+
bootstrapTokensCSRAutoApprovalClusterRoleBinding = "kubeadm:node-autoapprove-bootstrap"
34+
bootstrapTokensCSRAutoApprovalClusterRoleName = "system:certificates.k8s.io:certificatesigningrequests:nodeclient"
35+
)
36+
37+
// Define container for all the test specification aimed at verifying
38+
// that kubeadm creates the bootstrap token, the system:bootstrappers:kubeadm:default-node-token group
39+
// and that all the related RBAC rules are in place
40+
var _ = KubeadmDescribe("bootstrap token", func() {
41+
42+
// Get an instance of the k8s test framework
43+
f := framework.NewDefaultFramework("bootstrap token")
44+
45+
// Tests in this container are not expected to create new objects in the cluster
46+
// so we are disabling the creation of a namespace in order to get a faster execution
47+
f.SkipNamespaceCreation = true
48+
49+
It("should exist and be properly configured", func() {
50+
secrets, err := f.ClientSet.CoreV1().
51+
Secrets(kubeSystemNamespace).
52+
List(metav1.ListOptions{})
53+
framework.ExpectNoError(err, "error reading Secrets")
54+
55+
tokenNum := 0
56+
for _, s := range secrets.Items {
57+
if s.Type == corev1.SecretTypeBootstrapToken {
58+
//TODO: might be we want to further check tokens (auth-extra-groups, usage etc)
59+
tokenNum++
60+
}
61+
}
62+
Expect(tokenNum).Should(BeNumerically(">", 0), "At least one bootstrap token should exist")
63+
})
64+
65+
It("should be allowed to post CSR for kubelet certificates on joining nodes", func() {
66+
ExpectClusterRoleBindingWithSubjectAndRole(f.ClientSet,
67+
bootstrapTokensAllowPostCSRClusterRoleBinding,
68+
rbacv1.GroupKind, bootstrapTokensGroup,
69+
bootstrapTokensAllowPostCSRClusterRoleName,
70+
)
71+
//TODO: check if possible to verify "allowed to post CSR" using subject asses review as well
72+
})
73+
74+
It("should be allowed to auto approve CSR for kubelet certificates on joining nodes", func() {
75+
ExpectClusterRoleBindingWithSubjectAndRole(f.ClientSet,
76+
bootstrapTokensCSRAutoApprovalClusterRoleBinding,
77+
rbacv1.GroupKind, bootstrapTokensGroup,
78+
bootstrapTokensCSRAutoApprovalClusterRoleName,
79+
)
80+
//TODO: check if possible to verify "allowed to auto approve CSR" using subject asses review as well
81+
})
82+
})

‎test/e2e_kubeadm/cluster_info_test.go‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
/*
2+
Copyright 2019 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package e2e_kubeadm
18+
19+
import (
20+
authv1 "k8s.io/api/authorization/v1"
21+
rbacv1 "k8s.io/api/rbac/v1"
22+
bootstrapapi "k8s.io/cluster-bootstrap/token/api"
23+
"k8s.io/kubernetes/test/e2e/framework"
24+
25+
. "github.com/onsi/ginkgo"
26+
. "github.com/onsi/gomega"
27+
)
28+
29+
const (
30+
clusterInfoConfigMapName = "cluster-info"
31+
clusterInfoRoleName = "kubeadm:bootstrap-signer-clusterinfo"
32+
clusterInfoRoleBindingName = clusterInfoRoleName
33+
)
34+
35+
var (
36+
clusterInfoConfigMapResource = &authv1.ResourceAttributes{
37+
Namespace: kubePublicNamespace,
38+
Name: clusterInfoConfigMapName,
39+
Resource: "configmaps",
40+
Verb: "get",
41+
}
42+
)
43+
44+
// Define container for all the test specification aimed at verifying
45+
// that kubeadm creates the cluster-info ConfigMap, that it is properly configured
46+
// and that all the related RBAC rules are in place
47+
var _ = KubeadmDescribe("cluster-info ConfigMap", func() {
48+
49+
// Get an instance of the k8s test framework
50+
f := framework.NewDefaultFramework("cluster-info")
51+
52+
// Tests in this container are not expected to create new objects in the cluster
53+
// so we are disabling the creation of a namespace in order to get a faster execution
54+
f.SkipNamespaceCreation = true
55+
56+
It("should exist and be properly configured", func() {
57+
// Nb. this is technically implemented a part of the bootstrap-token phase
58+
cm := GetConfigMap(f.ClientSet, kubePublicNamespace, clusterInfoConfigMapName)
59+
60+
Expect(cm.Data).To(HaveKey(HavePrefix(bootstrapapi.JWSSignatureKeyPrefix)))
61+
Expect(cm.Data).To(HaveKey(bootstrapapi.KubeConfigKey))
62+
63+
//TODO: What else? server?
64+
})
65+
66+
It("should have related Role and RoleBinding", func() {
67+
// Nb. this is technically implemented a part of the bootstrap-token phase
68+
ExpectRole(f.ClientSet, kubePublicNamespace, clusterInfoRoleName)
69+
ExpectRoleBinding(f.ClientSet, kubePublicNamespace, clusterInfoRoleBindingName)
70+
})
71+
72+
It("should be accessible for anonymous", func() {
73+
ExpectSubjectHasAccessToResource(f.ClientSet,
74+
rbacv1.UserKind, anonymousUser,
75+
clusterInfoConfigMapResource,
76+
)
77+
})
78+
})

‎test/e2e_kubeadm/matchers.go‎ renamed to ‎test/e2e_kubeadm/const.go‎

Lines changed: 7 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
Copyright 2018 The Kubernetes Authors.
2+
Copyright 2019 The Kubernetes Authors.
33
44
Licensed under the Apache License, Version 2.0 (the "License");
55
you may not use this file except in compliance with the License.
@@ -16,22 +16,11 @@ limitations under the License.
1616

1717
package e2e_kubeadm
1818

19-
import (
20-
"github.com/onsi/gomega"
21-
"github.com/onsi/gomega/gstruct"
22-
corev1 "k8s.io/api/core/v1"
23-
)
19+
const (
20+
kubePublicNamespace = "kube-public"
21+
kubeSystemNamespace = "kube-system"
2422

25-
func subject(name, kind string) gomega.OmegaMatcher {
26-
return gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
27-
"Name": gomega.Equal(name),
28-
"Kind": gomega.Equal(kind),
29-
})
30-
}
23+
anonymousUser = "system:anonymous"
3124

32-
func taint(key string, effect corev1.TaintEffect) gomega.OmegaMatcher {
33-
return gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
34-
"Key": gomega.Equal(key),
35-
"Effect": gomega.Equal(effect),
36-
})
37-
}
25+
nodesGroup = "system:nodes"
26+
)

‎test/e2e_kubeadm/controlplane_nodes_test.go‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
/*
2+
Copyright 2019 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package e2e_kubeadm
18+
19+
import (
20+
corev1 "k8s.io/api/core/v1"
21+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
22+
"k8s.io/apimachinery/pkg/labels"
23+
clientset "k8s.io/client-go/kubernetes"
24+
"k8s.io/kubernetes/test/e2e/framework"
25+
26+
. "github.com/onsi/ginkgo"
27+
. "github.com/onsi/gomega"
28+
)
29+
30+
const (
31+
controlPlaneTaint = "node-role.kubernetes.io/master"
32+
)
33+
34+
// Define container for all the test specification aimed at verifying
35+
// that kubeadm configures the control-plane node as expected
36+
var _ = KubeadmDescribe("control-plane node", func() {
37+
38+
// Get an instance of the k8s test framework
39+
f := framework.NewDefaultFramework("control-plane node")
40+
41+
// Tests in this container are not expected to create new objects in the cluster
42+
// so we are disabling the creation of a namespace in order to get a faster execution
43+
f.SkipNamespaceCreation = true
44+
45+
// Important! please note that this test can't be run on single-node clusters
46+
// in case you can skip this test with SKIP=multi-node
47+
It("should be labelled and tainted [multi-node]", func() {
48+
// get all control-plane nodes (and this implicitly checks that node are properly labeled)
49+
controlPlanes := getControlPlaneNodes(f.ClientSet)
50+
51+
// checks if there is at least one control-plane node
52+
Expect(controlPlanes.Items).NotTo(BeEmpty(), "at least one node with label %s should exist. if you are running test on a single-node cluster, you can skip this test with SKIP=multi-node", controlPlaneTaint)
53+
54+
// checks that the control-plane nodes have the expected taint
55+
for _, cp := range controlPlanes.Items {
56+
framework.ExpectNodeHasTaint(f.ClientSet, cp.GetName(), &corev1.Taint{Key: controlPlaneTaint, Effect: corev1.TaintEffectNoSchedule})
57+
}
58+
})
59+
})
60+
61+
func getControlPlaneNodes(c clientset.Interface) *corev1.NodeList {
62+
selector := labels.Set{controlPlaneTaint: ""}.AsSelector()
63+
masters, err := c.CoreV1().Nodes().
64+
List(metav1.ListOptions{LabelSelector: selector.String()})
65+
framework.ExpectNoError(err, "error reading control-plane nodes")
66+
67+
return masters
68+
}

‎test/e2e_kubeadm/framework.go‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
/*
2+
Copyright 2019 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package e2e_kubeadm
18+
19+
import "k8s.io/kubernetes/test/e2e/framework"
20+
21+
func KubeadmDescribe(text string, body func()) bool {
22+
return framework.KubeDescribe("[sig-cluster-lifecycle] [area-kubeadm] "+text, body)
23+
}

0 commit comments

Comments
 (0)