Skip to content

Commit 074aba8

Browse files
cwdsuzhoucwdsuzhou
committed
Fix storage e2e clean up
1 parent 1c97410 commit 074aba8

File tree

14 files changed

+112
-71
lines changed

14 files changed

+112
-71
lines changed

‎test/e2e/storage/testsuites/BUILD‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ go_library(
5656
"//test/utils/image:go_default_library",
5757
"//vendor/github.com/onsi/ginkgo:go_default_library",
5858
"//vendor/github.com/onsi/gomega:go_default_library",
59+
"//vendor/github.com/pkg/errors:go_default_library",
5960
],
6061
)
6162

‎test/e2e/storage/testsuites/base.go‎

Lines changed: 36 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -26,13 +26,15 @@ import (
2626
"time"
2727

2828
"github.com/onsi/ginkgo"
29+
"github.com/pkg/errors"
2930

3031
v1 "k8s.io/api/core/v1"
3132
storagev1 "k8s.io/api/storage/v1"
3233
apierrs "k8s.io/apimachinery/pkg/api/errors"
3334
"k8s.io/apimachinery/pkg/api/resource"
3435
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3536
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
37+
apierrors "k8s.io/apimachinery/pkg/util/errors"
3638
utilerrors "k8s.io/apimachinery/pkg/util/errors"
3739
"k8s.io/apimachinery/pkg/util/sets"
3840
clientset "k8s.io/client-go/kubernetes"
@@ -81,7 +83,7 @@ type TestSuiteInfo struct {
8183
// TestResource represents an interface for resources that is used by TestSuite
8284
type TestResource interface {
8385
// cleanupResource cleans up the test resources created when setting up the resource
84-
cleanupResource()
86+
cleanupResource() error
8587
}
8688

8789
func getTestNameStr(suite TestSuite, pattern testpatterns.TestPattern) string {
@@ -268,9 +270,9 @@ func createVolumeSource(pvcName string, readOnly bool) *v1.VolumeSource {
268270
}
269271

270272
// cleanupResource cleans up genericVolumeTestResource
271-
func (r *genericVolumeTestResource) cleanupResource() {
273+
func (r *genericVolumeTestResource) cleanupResource() error {
272274
f := r.config.Framework
273-
275+
var cleanUpErrs []error
274276
if r.pvc != nil || r.pv != nil {
275277
switch r.pattern.VolType {
276278
case testpatterns.PreprovisionedPV:
@@ -287,10 +289,15 @@ func (r *genericVolumeTestResource) cleanupResource() {
287289
}
288290
if r.pvc != nil {
289291
err := e2epv.DeletePersistentVolumeClaim(f.ClientSet, r.pvc.Name, f.Namespace.Name)
290-
framework.ExpectNoError(err, "Failed to delete PVC %v", r.pvc.Name)
292+
if err != nil {
293+
cleanUpErrs = append(cleanUpErrs, errors.Wrapf(err, "Failed to delete PVC %v", r.pvc.Name))
294+
}
291295
if r.pv != nil {
292296
err = framework.WaitForPersistentVolumeDeleted(f.ClientSet, r.pv.Name, 5*time.Second, 5*time.Minute)
293-
framework.ExpectNoError(err, "Persistent Volume %v not deleted by dynamic provisioner", r.pv.Name)
297+
if err != nil {
298+
cleanUpErrs = append(cleanUpErrs, errors.Wrapf(err,
299+
"Persistent Volume %v not deleted by dynamic provisioner", r.pv.Name))
300+
}
294301
}
295302
}
296303
default:
@@ -300,13 +307,18 @@ func (r *genericVolumeTestResource) cleanupResource() {
300307

301308
if r.sc != nil {
302309
ginkgo.By("Deleting sc")
303-
deleteStorageClass(f.ClientSet, r.sc.Name)
310+
if err := deleteStorageClass(f.ClientSet, r.sc.Name); err != nil {
311+
cleanUpErrs = append(cleanUpErrs, errors.Wrapf(err, "Failed to delete StorageClass %v", r.sc.Name))
312+
}
304313
}
305314

306315
// Cleanup volume for pre-provisioned volume tests
307316
if r.volume != nil {
308-
r.volume.DeleteVolume()
317+
if err := tryFunc(r.volume.DeleteVolume); err != nil {
318+
cleanUpErrs = append(cleanUpErrs, errors.Wrap(err, "Failed to delete Volume"))
319+
}
309320
}
321+
return apierrors.NewAggregate(cleanUpErrs)
310322
}
311323

312324
func createPVCPV(
@@ -396,11 +408,12 @@ func isDelayedBinding(sc *storagev1.StorageClass) bool {
396408
}
397409

398410
// deleteStorageClass deletes the passed in StorageClass and catches errors other than "Not Found"
399-
func deleteStorageClass(cs clientset.Interface, className string) {
411+
func deleteStorageClass(cs clientset.Interface, className string) error {
400412
err := cs.StorageV1().StorageClasses().Delete(className, nil)
401413
if err != nil && !apierrs.IsNotFound(err) {
402-
framework.ExpectNoError(err)
414+
return err
403415
}
416+
return nil
404417
}
405418

406419
// convertTestConfig returns a framework test config with the
@@ -685,3 +698,17 @@ func skipVolTypePatterns(pattern testpatterns.TestPattern, driver TestDriver, sk
685698
framework.Skipf("Driver supports dynamic provisioning, skipping %s pattern", pattern.VolType)
686699
}
687700
}
701+
702+
func tryFunc(f func()) error {
703+
var err error
704+
if f == nil {
705+
return nil
706+
}
707+
defer func() {
708+
if recoverError := recover(); recoverError != nil {
709+
err = fmt.Errorf("%v", recoverError)
710+
}
711+
}()
712+
f()
713+
return err
714+
}

‎test/e2e/storage/testsuites/disruptive.go‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,9 @@ package testsuites
1818

1919
import (
2020
"github.com/onsi/ginkgo"
21+
2122
v1 "k8s.io/api/core/v1"
23+
errors "k8s.io/apimachinery/pkg/util/errors"
2224
clientset "k8s.io/client-go/kubernetes"
2325
"k8s.io/kubernetes/test/e2e/framework"
2426
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
@@ -97,22 +99,23 @@ func (s *disruptiveTestSuite) defineTests(driver TestDriver, pattern testpattern
9799
}
98100

99101
cleanup := func() {
102+
var errs []error
100103
if l.pod != nil {
101104
ginkgo.By("Deleting pod")
102105
err := e2epod.DeletePodWithWait(f.ClientSet, l.pod)
103-
framework.ExpectNoError(err, "while deleting pod")
106+
errs = append(errs, err)
104107
l.pod = nil
105108
}
106109

107110
if l.resource != nil {
108-
l.resource.cleanupResource()
111+
err := l.resource.cleanupResource()
112+
errs = append(errs, err)
109113
l.resource = nil
110114
}
111115

112-
if l.driverCleanup != nil {
113-
l.driverCleanup()
114-
l.driverCleanup = nil
115-
}
116+
errs = append(errs, tryFunc(l.driverCleanup))
117+
l.driverCleanup = nil
118+
framework.ExpectNoError(errors.NewAggregate(errs), "while cleaning up resource")
116119
}
117120

118121
type testBody func(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod)

‎test/e2e/storage/testsuites/ephemeral.go‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -107,10 +107,9 @@ func (p *ephemeralTestSuite) defineTests(driver TestDriver, pattern testpatterns
107107
}
108108

109109
cleanup := func() {
110-
if l.driverCleanup != nil {
111-
l.driverCleanup()
112-
l.driverCleanup = nil
113-
}
110+
err := tryFunc(l.driverCleanup)
111+
framework.ExpectNoError(err, "while cleaning up driver")
112+
l.driverCleanup = nil
114113
}
115114

116115
ginkgo.It("should create read-only inline ephemeral volume", func() {

‎test/e2e/storage/testsuites/multivolume.go‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424

2525
v1 "k8s.io/api/core/v1"
2626
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
27+
"k8s.io/apimachinery/pkg/util/errors"
2728
clientset "k8s.io/client-go/kubernetes"
2829
"k8s.io/kubernetes/test/e2e/framework"
2930
e2enode "k8s.io/kubernetes/test/e2e/framework/node"
@@ -109,15 +110,14 @@ func (t *multiVolumeTestSuite) defineTests(driver TestDriver, pattern testpatter
109110
}
110111

111112
cleanup := func() {
113+
var errs []error
112114
for _, resource := range l.resources {
113-
resource.cleanupResource()
114-
}
115-
116-
if l.driverCleanup != nil {
117-
l.driverCleanup()
118-
l.driverCleanup = nil
115+
errs = append(errs, resource.cleanupResource())
119116
}
120117

118+
errs = append(errs, tryFunc(l.driverCleanup))
119+
l.driverCleanup = nil
120+
framework.ExpectNoError(errors.NewAggregate(errs), "while cleanup resource")
121121
validateMigrationVolumeOpCounts(f.ClientSet, dInfo.InTreePluginName, l.intreeOps, l.migratedOps)
122122
}
123123

‎test/e2e/storage/testsuites/provisioning.go‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -162,10 +162,9 @@ func (p *provisioningTestSuite) defineTests(driver TestDriver, pattern testpatte
162162
}
163163

164164
cleanup := func() {
165-
if l.driverCleanup != nil {
166-
l.driverCleanup()
167-
l.driverCleanup = nil
168-
}
165+
err := tryFunc(l.driverCleanup)
166+
l.driverCleanup = nil
167+
framework.ExpectNoError(err, "while cleaning up driver")
169168

170169
validateMigrationVolumeOpCounts(f.ClientSet, dInfo.InTreePluginName, l.intreeOps, l.migratedOps)
171170
}
@@ -454,7 +453,10 @@ func (t StorageClassTest) TestBindingWaitForFirstConsumerMultiPVC(claims []*v1.P
454453
ginkgo.By("creating a storage class " + t.Class.Name)
455454
class, err := t.Client.StorageV1().StorageClasses().Create(t.Class)
456455
framework.ExpectNoError(err)
457-
defer deleteStorageClass(t.Client, class.Name)
456+
defer func() {
457+
err = deleteStorageClass(t.Client, class.Name)
458+
framework.ExpectNoError(err, "While deleting storage class")
459+
}()
458460

459461
ginkgo.By("creating claims")
460462
var claimNames []string

‎test/e2e/storage/testsuites/snapshottable.go‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,10 @@ func (s *snapshottableTestSuite) defineTests(driver TestDriver, pattern testpatt
106106

107107
// Now do the more expensive test initialization.
108108
config, driverCleanup := driver.PrepareTest(f)
109-
defer driverCleanup()
109+
defer func() {
110+
err := tryFunc(driverCleanup)
111+
framework.ExpectNoError(err, "while cleaning up driver")
112+
}()
110113

111114
vsc := sDriver.GetSnapshotClass(config)
112115
class := dDriver.GetDynamicProvisionStorageClass(config, "")

‎test/e2e/storage/testsuites/subpath.go‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,14 @@ import (
2121
"path/filepath"
2222
"regexp"
2323
"strings"
24+
"time"
25+
26+
"github.com/onsi/ginkgo"
27+
"github.com/onsi/gomega"
2428

2529
v1 "k8s.io/api/core/v1"
2630
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
31+
"k8s.io/apimachinery/pkg/util/errors"
2732
"k8s.io/apimachinery/pkg/util/rand"
2833
"k8s.io/apimachinery/pkg/util/sets"
2934
"k8s.io/apimachinery/pkg/util/wait"
@@ -34,11 +39,6 @@ import (
3439
"k8s.io/kubernetes/test/e2e/storage/testpatterns"
3540
"k8s.io/kubernetes/test/e2e/storage/utils"
3641
imageutils "k8s.io/kubernetes/test/utils/image"
37-
38-
"time"
39-
40-
"github.com/onsi/ginkgo"
41-
"github.com/onsi/gomega"
4242
)
4343

4444
var (
@@ -162,22 +162,22 @@ func (s *subPathTestSuite) defineTests(driver TestDriver, pattern testpatterns.T
162162
}
163163

164164
cleanup := func() {
165+
var errs []error
165166
if l.pod != nil {
166167
ginkgo.By("Deleting pod")
167168
err := e2epod.DeletePodWithWait(f.ClientSet, l.pod)
168-
framework.ExpectNoError(err, "while deleting pod")
169+
errs = append(errs, err)
169170
l.pod = nil
170171
}
171172

172173
if l.resource != nil {
173-
l.resource.cleanupResource()
174+
errs = append(errs, l.resource.cleanupResource())
174175
l.resource = nil
175176
}
176177

177-
if l.driverCleanup != nil {
178-
l.driverCleanup()
179-
l.driverCleanup = nil
180-
}
178+
errs = append(errs, tryFunc(l.driverCleanup))
179+
l.driverCleanup = nil
180+
framework.ExpectNoError(errors.NewAggregate(errs), "while cleaning up resource")
181181

182182
if l.hostExec != nil {
183183
l.hostExec.Cleanup()

‎test/e2e/storage/testsuites/topology.go‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -154,9 +154,9 @@ func (t *topologyTestSuite) defineTests(driver TestDriver, pattern testpatterns.
154154

155155
cleanup := func(l topologyTest) {
156156
t.cleanupResources(cs, &l)
157-
if l.driverCleanup != nil {
158-
l.driverCleanup()
159-
}
157+
err := tryFunc(l.driverCleanup)
158+
l.driverCleanup = nil
159+
framework.ExpectNoError(err, "while cleaning up driver")
160160

161161
validateMigrationVolumeOpCounts(f.ClientSet, dInfo.InTreePluginName, l.intreeOps, l.migratedOps)
162162
}
@@ -353,5 +353,7 @@ func (t *topologyTestSuite) cleanupResources(cs clientset.Interface, l *topology
353353
err := e2epod.DeletePodWithWait(cs, l.pod)
354354
framework.ExpectNoError(err, "while deleting pod")
355355
}
356-
l.resource.cleanupResource()
356+
357+
err := l.resource.cleanupResource()
358+
framework.ExpectNoError(err, "while clean up resource")
357359
}

‎test/e2e/storage/testsuites/volume_expand.go‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import (
2626
v1 "k8s.io/api/core/v1"
2727
"k8s.io/apimachinery/pkg/api/resource"
2828
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
29+
"k8s.io/apimachinery/pkg/util/errors"
2930
"k8s.io/apimachinery/pkg/util/wait"
3031
clientset "k8s.io/client-go/kubernetes"
3132
"k8s.io/kubernetes/test/e2e/framework"
@@ -113,30 +114,29 @@ func (v *volumeExpandTestSuite) defineTests(driver TestDriver, pattern testpatte
113114
}
114115

115116
cleanup := func() {
117+
var errs []error
116118
if l.pod != nil {
117119
ginkgo.By("Deleting pod")
118120
err := e2epod.DeletePodWithWait(f.ClientSet, l.pod)
119-
framework.ExpectNoError(err, "while deleting pod")
121+
errs = append(errs, err)
120122
l.pod = nil
121123
}
122124

123125
if l.pod2 != nil {
124126
ginkgo.By("Deleting pod2")
125127
err := e2epod.DeletePodWithWait(f.ClientSet, l.pod2)
126-
framework.ExpectNoError(err, "while deleting pod2")
128+
errs = append(errs, err)
127129
l.pod2 = nil
128130
}
129131

130132
if l.resource != nil {
131-
l.resource.cleanupResource()
133+
errs = append(errs, l.resource.cleanupResource())
132134
l.resource = nil
133135
}
134136

135-
if l.driverCleanup != nil {
136-
l.driverCleanup()
137-
l.driverCleanup = nil
138-
}
139-
137+
errs = append(errs, tryFunc(l.driverCleanup))
138+
l.driverCleanup = nil
139+
framework.ExpectNoError(errors.NewAggregate(errs), "while cleaning up resource")
140140
validateMigrationVolumeOpCounts(f.ClientSet, driver.GetDriverInfo().InTreePluginName, l.intreeOps, l.migratedOps)
141141
}
142142

0 commit comments

Comments
 (0)