Skip to content

Commit 82c7c86

Browse files
authored
Merge pull request kubernetes#86169 from clarklee92/move-e2e/framework/statefulset
e2e: move funs of framework/statefulset to e2e/apps & e2e/upgrades
2 parents aab1bef + 623c4f9 commit 82c7c86

File tree

8 files changed

+361
-337
lines changed

8 files changed

+361
-337
lines changed

test/e2e/apps/BUILD

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ go_library(
2020
"replica_set.go",
2121
"statefulset.go",
2222
"types.go",
23+
"wait.go",
2324
],
2425
importpath = "k8s.io/kubernetes/test/e2e/apps",
2526
deps = [

test/e2e/apps/statefulset.go

Lines changed: 186 additions & 61 deletions
Large diffs are not rendered by default.

test/e2e/apps/wait.go

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
/*
2+
Copyright 2019 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package apps
18+
19+
import (
20+
appsv1 "k8s.io/api/apps/v1"
21+
v1 "k8s.io/api/core/v1"
22+
clientset "k8s.io/client-go/kubernetes"
23+
24+
podutil "k8s.io/kubernetes/pkg/api/v1/pod"
25+
"k8s.io/kubernetes/test/e2e/framework"
26+
e2esset "k8s.io/kubernetes/test/e2e/framework/statefulset"
27+
)
28+
29+
// waitForPartitionedRollingUpdate waits for all Pods in set to exist and have the correct revision. set must have
30+
// a RollingUpdateStatefulSetStrategyType with a non-nil RollingUpdate and Partition. All Pods with ordinals less
31+
// than or equal to the Partition are expected to be at set's current revision. All other Pods are expected to be
32+
// at its update revision.
33+
func waitForPartitionedRollingUpdate(c clientset.Interface, set *appsv1.StatefulSet) (*appsv1.StatefulSet, *v1.PodList) {
34+
var pods *v1.PodList
35+
if set.Spec.UpdateStrategy.Type != appsv1.RollingUpdateStatefulSetStrategyType {
36+
framework.Failf("StatefulSet %s/%s attempt to wait for partitioned update with updateStrategy %s",
37+
set.Namespace,
38+
set.Name,
39+
set.Spec.UpdateStrategy.Type)
40+
}
41+
if set.Spec.UpdateStrategy.RollingUpdate == nil || set.Spec.UpdateStrategy.RollingUpdate.Partition == nil {
42+
framework.Failf("StatefulSet %s/%s attempt to wait for partitioned update with nil RollingUpdate or nil Partition",
43+
set.Namespace,
44+
set.Name)
45+
}
46+
e2esset.WaitForState(c, set, func(set2 *appsv1.StatefulSet, pods2 *v1.PodList) (bool, error) {
47+
set = set2
48+
pods = pods2
49+
partition := int(*set.Spec.UpdateStrategy.RollingUpdate.Partition)
50+
if len(pods.Items) < int(*set.Spec.Replicas) {
51+
return false, nil
52+
}
53+
if partition <= 0 && set.Status.UpdateRevision != set.Status.CurrentRevision {
54+
framework.Logf("Waiting for StatefulSet %s/%s to complete update",
55+
set.Namespace,
56+
set.Name,
57+
)
58+
e2esset.SortStatefulPods(pods)
59+
for i := range pods.Items {
60+
if pods.Items[i].Labels[appsv1.StatefulSetRevisionLabel] != set.Status.UpdateRevision {
61+
framework.Logf("Waiting for Pod %s/%s to have revision %s update revision %s",
62+
pods.Items[i].Namespace,
63+
pods.Items[i].Name,
64+
set.Status.UpdateRevision,
65+
pods.Items[i].Labels[appsv1.StatefulSetRevisionLabel])
66+
}
67+
}
68+
return false, nil
69+
}
70+
for i := int(*set.Spec.Replicas) - 1; i >= partition; i-- {
71+
if pods.Items[i].Labels[appsv1.StatefulSetRevisionLabel] != set.Status.UpdateRevision {
72+
framework.Logf("Waiting for Pod %s/%s to have revision %s update revision %s",
73+
pods.Items[i].Namespace,
74+
pods.Items[i].Name,
75+
set.Status.UpdateRevision,
76+
pods.Items[i].Labels[appsv1.StatefulSetRevisionLabel])
77+
return false, nil
78+
}
79+
}
80+
return true, nil
81+
})
82+
return set, pods
83+
}
84+
85+
// waitForStatus waits for the StatefulSetStatus's ObservedGeneration to be greater than or equal to set's Generation.
86+
// The returned StatefulSet contains such a StatefulSetStatus
87+
func waitForStatus(c clientset.Interface, set *appsv1.StatefulSet) *appsv1.StatefulSet {
88+
e2esset.WaitForState(c, set, func(set2 *appsv1.StatefulSet, pods *v1.PodList) (bool, error) {
89+
if set2.Status.ObservedGeneration >= set.Generation {
90+
set = set2
91+
return true, nil
92+
}
93+
return false, nil
94+
})
95+
return set
96+
}
97+
98+
// waitForPodNotReady waits for the Pod named podName in set to exist and to not have a Ready condition.
99+
func waitForPodNotReady(c clientset.Interface, set *appsv1.StatefulSet, podName string) (*appsv1.StatefulSet, *v1.PodList) {
100+
var pods *v1.PodList
101+
e2esset.WaitForState(c, set, func(set2 *appsv1.StatefulSet, pods2 *v1.PodList) (bool, error) {
102+
set = set2
103+
pods = pods2
104+
for i := range pods.Items {
105+
if pods.Items[i].Name == podName {
106+
return !podutil.IsPodReady(&pods.Items[i]), nil
107+
}
108+
}
109+
return false, nil
110+
})
111+
return set, pods
112+
}
113+
114+
// waitForRollingUpdate waits for all Pods in set to exist and have the correct revision and for the RollingUpdate to
115+
// complete. set must have a RollingUpdateStatefulSetStrategyType.
116+
func waitForRollingUpdate(c clientset.Interface, set *appsv1.StatefulSet) (*appsv1.StatefulSet, *v1.PodList) {
117+
var pods *v1.PodList
118+
if set.Spec.UpdateStrategy.Type != appsv1.RollingUpdateStatefulSetStrategyType {
119+
framework.Failf("StatefulSet %s/%s attempt to wait for rolling update with updateStrategy %s",
120+
set.Namespace,
121+
set.Name,
122+
set.Spec.UpdateStrategy.Type)
123+
}
124+
e2esset.WaitForState(c, set, func(set2 *appsv1.StatefulSet, pods2 *v1.PodList) (bool, error) {
125+
set = set2
126+
pods = pods2
127+
if len(pods.Items) < int(*set.Spec.Replicas) {
128+
return false, nil
129+
}
130+
if set.Status.UpdateRevision != set.Status.CurrentRevision {
131+
framework.Logf("Waiting for StatefulSet %s/%s to complete update",
132+
set.Namespace,
133+
set.Name,
134+
)
135+
e2esset.SortStatefulPods(pods)
136+
for i := range pods.Items {
137+
if pods.Items[i].Labels[appsv1.StatefulSetRevisionLabel] != set.Status.UpdateRevision {
138+
framework.Logf("Waiting for Pod %s/%s to have revision %s update revision %s",
139+
pods.Items[i].Namespace,
140+
pods.Items[i].Name,
141+
set.Status.UpdateRevision,
142+
pods.Items[i].Labels[appsv1.StatefulSetRevisionLabel])
143+
}
144+
}
145+
return false, nil
146+
}
147+
return true, nil
148+
})
149+
return set, pods
150+
}
151+
152+
// waitForRunningAndNotReady waits for numStatefulPods in ss to be Running and not Ready.
153+
func waitForRunningAndNotReady(c clientset.Interface, numStatefulPods int32, ss *appsv1.StatefulSet) {
154+
e2esset.WaitForRunning(c, numStatefulPods, 0, ss)
155+
}

test/e2e/framework/statefulset/BUILD

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@ go_library(
1818
"//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
1919
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
2020
"//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library",
21-
"//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
2221
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
2322
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
2423
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",

test/e2e/framework/statefulset/fixtures.go

Lines changed: 0 additions & 85 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@ import (
2727
v1 "k8s.io/api/core/v1"
2828
"k8s.io/apimachinery/pkg/api/resource"
2929
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
30-
"k8s.io/apimachinery/pkg/util/intstr"
3130
clientset "k8s.io/client-go/kubernetes"
3231
podutil "k8s.io/kubernetes/pkg/api/v1/pod"
3332
e2efwk "k8s.io/kubernetes/test/e2e/framework"
@@ -112,95 +111,11 @@ func NewStatefulSetPVC(name string) v1.PersistentVolumeClaim {
112111
}
113112
}
114113

115-
// CreateStatefulSetService creates a Headless Service with Name name and Selector set to match labels.
116-
func CreateStatefulSetService(name string, labels map[string]string) *v1.Service {
117-
headlessService := &v1.Service{
118-
ObjectMeta: metav1.ObjectMeta{
119-
Name: name,
120-
},
121-
Spec: v1.ServiceSpec{
122-
Selector: labels,
123-
},
124-
}
125-
headlessService.Spec.Ports = []v1.ServicePort{
126-
{Port: 80, Name: "http", Protocol: v1.ProtocolTCP},
127-
}
128-
headlessService.Spec.ClusterIP = "None"
129-
return headlessService
130-
}
131-
132-
// SetHTTPProbe sets the pod template's ReadinessProbe for Webserver StatefulSet containers.
133-
// This probe can then be controlled with BreakHTTPProbe() and RestoreHTTPProbe().
134-
// Note that this cannot be used together with PauseNewPods().
135-
func SetHTTPProbe(ss *appsv1.StatefulSet) {
136-
ss.Spec.Template.Spec.Containers[0].ReadinessProbe = httpProbe
137-
}
138-
139-
// BreakHTTPProbe breaks the readiness probe for Nginx StatefulSet containers in ss.
140-
func BreakHTTPProbe(c clientset.Interface, ss *appsv1.StatefulSet) error {
141-
path := httpProbe.HTTPGet.Path
142-
if path == "" {
143-
return fmt.Errorf("path expected to be not empty: %v", path)
144-
}
145-
// Ignore 'mv' errors to make this idempotent.
146-
cmd := fmt.Sprintf("mv -v /usr/local/apache2/htdocs%v /tmp/ || true", path)
147-
return ExecInStatefulPods(c, ss, cmd)
148-
}
149-
150-
// BreakPodHTTPProbe breaks the readiness probe for Nginx StatefulSet containers in one pod.
151-
func BreakPodHTTPProbe(ss *appsv1.StatefulSet, pod *v1.Pod) error {
152-
path := httpProbe.HTTPGet.Path
153-
if path == "" {
154-
return fmt.Errorf("path expected to be not empty: %v", path)
155-
}
156-
// Ignore 'mv' errors to make this idempotent.
157-
cmd := fmt.Sprintf("mv -v /usr/local/apache2/htdocs%v /tmp/ || true", path)
158-
stdout, err := e2efwk.RunHostCmdWithRetries(pod.Namespace, pod.Name, cmd, StatefulSetPoll, StatefulPodTimeout)
159-
e2efwk.Logf("stdout of %v on %v: %v", cmd, pod.Name, stdout)
160-
return err
161-
}
162-
163-
// RestoreHTTPProbe restores the readiness probe for Nginx StatefulSet containers in ss.
164-
func RestoreHTTPProbe(c clientset.Interface, ss *appsv1.StatefulSet) error {
165-
path := httpProbe.HTTPGet.Path
166-
if path == "" {
167-
return fmt.Errorf("path expected to be not empty: %v", path)
168-
}
169-
// Ignore 'mv' errors to make this idempotent.
170-
cmd := fmt.Sprintf("mv -v /tmp%v /usr/local/apache2/htdocs/ || true", path)
171-
return ExecInStatefulPods(c, ss, cmd)
172-
}
173-
174-
// RestorePodHTTPProbe restores the readiness probe for Nginx StatefulSet containers in pod.
175-
func RestorePodHTTPProbe(ss *appsv1.StatefulSet, pod *v1.Pod) error {
176-
path := httpProbe.HTTPGet.Path
177-
if path == "" {
178-
return fmt.Errorf("path expected to be not empty: %v", path)
179-
}
180-
// Ignore 'mv' errors to make this idempotent.
181-
cmd := fmt.Sprintf("mv -v /tmp%v /usr/local/apache2/htdocs/ || true", path)
182-
stdout, err := e2efwk.RunHostCmdWithRetries(pod.Namespace, pod.Name, cmd, StatefulSetPoll, StatefulPodTimeout)
183-
e2efwk.Logf("stdout of %v on %v: %v", cmd, pod.Name, stdout)
184-
return err
185-
}
186-
187114
func hasPauseProbe(pod *v1.Pod) bool {
188115
probe := pod.Spec.Containers[0].ReadinessProbe
189116
return probe != nil && reflect.DeepEqual(probe.Exec.Command, pauseProbe.Exec.Command)
190117
}
191118

192-
var httpProbe = &v1.Probe{
193-
Handler: v1.Handler{
194-
HTTPGet: &v1.HTTPGetAction{
195-
Path: "/index.html",
196-
Port: intstr.IntOrString{IntVal: 80},
197-
},
198-
},
199-
PeriodSeconds: 1,
200-
SuccessThreshold: 1,
201-
FailureThreshold: 1,
202-
}
203-
204119
var pauseProbe = &v1.Probe{
205120
Handler: v1.Handler{
206121
Exec: &v1.ExecAction{Command: []string{"test", "-f", "/data/statefulset-continue"}},

test/e2e/framework/statefulset/rest.go

Lines changed: 0 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -68,15 +68,6 @@ func GetPodList(c clientset.Interface, ss *appsv1.StatefulSet) *v1.PodList {
6868
return podList
6969
}
7070

71-
// DeleteStatefulPodAtIndex deletes the Pod with ordinal index in ss.
72-
func DeleteStatefulPodAtIndex(c clientset.Interface, index int, ss *appsv1.StatefulSet) {
73-
name := getStatefulSetPodNameAtIndex(index, ss)
74-
noGrace := int64(0)
75-
if err := c.CoreV1().Pods(ss.Namespace).Delete(name, &metav1.DeleteOptions{GracePeriodSeconds: &noGrace}); err != nil {
76-
e2efwk.Failf("Failed to delete stateful pod %v for StatefulSet %v/%v: %v", name, ss.Namespace, ss.Name, err)
77-
}
78-
}
79-
8071
// DeleteAllStatefulSets deletes all StatefulSet API Objects in Namespace ns.
8172
func DeleteAllStatefulSets(c clientset.Interface, ns string) {
8273
ssList, err := c.AppsV1().StatefulSets(ns).List(metav1.ListOptions{LabelSelector: labels.Everything().String()})
@@ -149,29 +140,6 @@ func DeleteAllStatefulSets(c clientset.Interface, ns string) {
149140
}
150141
}
151142

152-
// UpdateStatefulSetWithRetries updates statfulset template with retries.
153-
func UpdateStatefulSetWithRetries(c clientset.Interface, namespace, name string, applyUpdate updateStatefulSetFunc) (statefulSet *appsv1.StatefulSet, err error) {
154-
statefulSets := c.AppsV1().StatefulSets(namespace)
155-
var updateErr error
156-
pollErr := wait.Poll(10*time.Millisecond, 1*time.Minute, func() (bool, error) {
157-
if statefulSet, err = statefulSets.Get(name, metav1.GetOptions{}); err != nil {
158-
return false, err
159-
}
160-
// Apply the update, then attempt to push it to the apiserver.
161-
applyUpdate(statefulSet)
162-
if statefulSet, err = statefulSets.Update(statefulSet); err == nil {
163-
e2efwk.Logf("Updating stateful set %s", name)
164-
return true, nil
165-
}
166-
updateErr = err
167-
return false, nil
168-
})
169-
if pollErr == wait.ErrWaitTimeout {
170-
pollErr = fmt.Errorf("couldn't apply the provided updated to stateful set %q: %v", name, updateErr)
171-
}
172-
return statefulSet, pollErr
173-
}
174-
175143
// Scale scales ss to count replicas.
176144
func Scale(c clientset.Interface, ss *appsv1.StatefulSet, count int32) (*appsv1.StatefulSet, error) {
177145
name := ss.Name
@@ -218,15 +186,6 @@ func Restart(c clientset.Interface, ss *appsv1.StatefulSet) {
218186
update(c, ss.Namespace, ss.Name, func(ss *appsv1.StatefulSet) { *(ss.Spec.Replicas) = oldReplicas })
219187
}
220188

221-
// GetStatefulSet gets the StatefulSet named name in namespace.
222-
func GetStatefulSet(c clientset.Interface, namespace, name string) *appsv1.StatefulSet {
223-
ss, err := c.AppsV1().StatefulSets(namespace).Get(name, metav1.GetOptions{})
224-
if err != nil {
225-
e2efwk.Failf("Failed to get StatefulSet %s/%s: %v", namespace, name, err)
226-
}
227-
return ss
228-
}
229-
230189
// CheckHostname verifies that all Pods in ss have the correct Hostname. If the returned error is not nil than verification failed.
231190
func CheckHostname(c clientset.Interface, ss *appsv1.StatefulSet) error {
232191
cmd := "printf $(hostname)"
@@ -285,19 +244,6 @@ func ExecInStatefulPods(c clientset.Interface, ss *appsv1.StatefulSet, cmd strin
285244
return nil
286245
}
287246

288-
type updateStatefulSetFunc func(*appsv1.StatefulSet)
289-
290-
// VerifyStatefulPodFunc is a func that examines a StatefulSetPod.
291-
type VerifyStatefulPodFunc func(*v1.Pod)
292-
293-
// VerifyPodAtIndex applies a visitor pattern to the Pod at index in ss. verify is applied to the Pod to "visit" it.
294-
func VerifyPodAtIndex(c clientset.Interface, index int, ss *appsv1.StatefulSet, verify VerifyStatefulPodFunc) {
295-
name := getStatefulSetPodNameAtIndex(index, ss)
296-
pod, err := c.CoreV1().Pods(ss.Namespace).Get(name, metav1.GetOptions{})
297-
e2efwk.ExpectNoError(err, fmt.Sprintf("Failed to get stateful pod %s for StatefulSet %s/%s", name, ss.Namespace, ss.Name))
298-
verify(pod)
299-
}
300-
301247
// udpate updates a statefulset, and it is only used within rest.go
302248
func update(c clientset.Interface, ns, name string, update func(ss *appsv1.StatefulSet)) *appsv1.StatefulSet {
303249
for i := 0; i < 3; i++ {
@@ -317,10 +263,3 @@ func update(c clientset.Interface, ns, name string, update func(ss *appsv1.State
317263
e2efwk.Failf("too many retries draining statefulset %q", name)
318264
return nil
319265
}
320-
321-
// getStatefulSetPodNameAtIndex gets formated pod name given index.
322-
func getStatefulSetPodNameAtIndex(index int, ss *appsv1.StatefulSet) string {
323-
// TODO: we won't use "-index" as the name strategy forever,
324-
// pull the name out from an identity mapper.
325-
return fmt.Sprintf("%v-%v", ss.Name, index)
326-
}

0 commit comments

Comments
 (0)