fixed e2e: wait pod deleted

Author: polefishu

test/e2e/drc/drc_test.go

@@ -37,6 +37,7 @@ var _ = Describe("DistributedRedisCluster CRUD", func() {
 		})
 		It("should recover from accidentally deleting master pods", func() {
 			e2e.DeleteMasterPodForDRC(drc, f.Client)
+			Eventually(e2e.IsDRCPodBeDeleted(f, drc), "5m", "10s").ShouldNot(HaveOccurred())
 			Eventually(e2e.IsDistributedRedisClusterProperly(f, drc), "10m", "10s").ShouldNot(HaveOccurred())
 			Expect(e2e.IsDBSizeConsistent(dbsize, goredis)).NotTo(HaveOccurred())
 		})

test/e2e/drcb/drcb_test.go

@@ -48,6 +48,7 @@ var _ = Describe("Restore DistributedRedisCluster From RedisClusterBackup", func
 				rdrc = e2e.RestoreDRC(drc, drcb)
 				Ω(f.CreateRedisCluster(rdrc)).Should(Succeed())
 				Eventually(e2e.IsDistributedRedisClusterProperly(f, rdrc), "10m", "10s").ShouldNot(HaveOccurred())
+				goredis = e2e.NewGoRedisClient(rdrc.Name, f.Namespace(), goredis.Password())
 				Expect(e2e.IsDBSizeConsistent(dbsize, goredis)).NotTo(HaveOccurred())
 			})
 			Context("when restore is succeeded", func() {
@@ -59,6 +60,7 @@ var _ = Describe("Restore DistributedRedisCluster From RedisClusterBackup", func
 				})
 				It("should recover from accidentally deleting master pods", func() {
 					e2e.DeleteMasterPodForDRC(rdrc, f.Client)
+					Eventually(e2e.IsDRCPodBeDeleted(f, rdrc), "5m", "10s").ShouldNot(HaveOccurred())
 					Eventually(e2e.IsDistributedRedisClusterProperly(f, rdrc), "10m", "10s").ShouldNot(HaveOccurred())
 					Expect(e2e.IsDBSizeConsistent(dbsize, goredis)).NotTo(HaveOccurred())
 				})

test/e2e/goredis_util.go

@@ -4,20 +4,31 @@ import (
 	"github.com/go-redis/redis"
 	uuid "github.com/satori/go.uuid"
 	"golang.org/x/sync/errgroup"
+	"time"
 )
 
+const defaultTimeOut = time.Second * 2
+
 // GoRedis contains ClusterClient.
 type GoRedis struct {
-	client *redis.ClusterClient
+	client   *redis.ClusterClient
+	password string
 }
 
 // NewGoRedis return a new ClusterClient.
 func NewGoRedis(addr, password string) *GoRedis {
 	return &GoRedis{
 		client: redis.NewClusterClient(&redis.ClusterOptions{
-			Addrs:    []string{addr},
-			Password: password,
+			Addrs:      []string{addr},
+			Password:   password,
+			MaxRetries: 5,
+
+			PoolSize:     3,
+			MinIdleConns: 1,
+			PoolTimeout:  defaultTimeOut,
+			IdleTimeout:  defaultTimeOut,
 		}),
+		password: password,
 	}
 }
 
@@ -45,3 +56,8 @@ func (g *GoRedis) StuffingData(round, n int) error {
 func (g *GoRedis) DBSize() (int64, error) {
 	return g.client.DBSize().Result()
 }
+
+// Password return redis password.
+func (g *GoRedis) Password() string {
+	return g.password
+}

test/e2e/operator_util.go

@@ -122,6 +122,9 @@ func IsDistributedRedisClusterProperly(f *Framework, drc *redisv1alpha1.Distribu
 			return err
 		}
 		if result.Status.Status != redisv1alpha1.ClusterStatusOK {
+			if result.Status.Status == redisv1alpha1.ClusterStatusKO {
+				f.Logf("DistributedRedisCluster %s is %s, reason: %s", drc.Name, result.Status.Status, result.Status.Reason)
+			}
 			return LogAndReturnErrorf("DistributedRedisCluster %s status not healthy, current: %s", drc.Name, result.Status.Status)
 		}
 		stsList, err := f.GetDRCStatefulSetByLabels(getLabels(drc))
@@ -177,6 +180,7 @@ func IsDistributedRedisClusterProperly(f *Framework, drc *redisv1alpha1.Distribu
 			}
 		}
 
+		drc.Spec = result.Spec
 		return nil
 	}
 }
@@ -299,6 +303,21 @@ func DeleteMasterPodForDRC(drc *redisv1alpha1.DistributedRedisCluster, client cl
 	}
 }
 
+func IsDRCPodBeDeleted(f *Framework, drc *redisv1alpha1.DistributedRedisCluster) func() error {
+	return func() error {
+		stsList, err := f.GetDRCStatefulSetByLabels(getLabels(drc))
+		if err != nil {
+			return LogAndReturnErrorf("GetDRCStatefulSetByLabels err: %s", err)
+		}
+		for _, sts := range stsList.Items {
+			if sts.Status.ReadyReplicas != (drc.Spec.ClusterReplicas + 1) {
+				return nil
+			}
+		}
+		return LogAndReturnErrorf("StatefulSet's Pod still running")
+	}
+}
+
 func NewRedisClusterBackup(name, namespace, image, drcName, storageSecretName, s3Endpoint, s3Bucket string) *redisv1alpha1.RedisClusterBackup {
 	return &redisv1alpha1.RedisClusterBackup{
 		ObjectMeta: metav1.ObjectMeta{

test/e2e/util.go

@@ -24,7 +24,7 @@ func Logf(format string, args ...interface{}) {
 // Failf reports a failure in the current e2e
 func Failf(format string, args ...interface{}) {
 	msg := fmt.Sprintf(format, args...)
-	log("INFO", msg)
+	log("ERROR", msg)
 	ginkgo.Fail(nowStamp()+": "+msg, 1)
 }