do not create StatefulSet pods when PVC is being deleted

Pod with PVC will not be scheduled if the PVC is being deleted.
This can happen when the PVC has finalizers of storage plugins.

Such a pod becomes pending.  Unfortunately, after the finalizer
finishes and PVC is deleted, the pod remains pending forever.
The StatefulSet controller does nothing for this pending pod.

This commit prevents the StatefulSet controller from creating
such pods when PVC is to be deleted.

Author: ymmt2005

pkg/controller/statefulset/stateful_pod_control.go

@@ -181,7 +181,7 @@ func (spc *realStatefulPodControl) recordClaimEvent(verb string, set *apps.State
 func (spc *realStatefulPodControl) createPersistentVolumeClaims(set *apps.StatefulSet, pod *v1.Pod) error {
 	var errs []error
 	for _, claim := range getPersistentVolumeClaims(set, pod) {
-		_, err := spc.pvcLister.PersistentVolumeClaims(claim.Namespace).Get(claim.Name)
+		pvc, err := spc.pvcLister.PersistentVolumeClaims(claim.Namespace).Get(claim.Name)
 		switch {
 		case apierrors.IsNotFound(err):
 			_, err := spc.client.CoreV1().PersistentVolumeClaims(claim.Namespace).Create(context.TODO(), &claim, metav1.CreateOptions{})
@@ -194,6 +194,10 @@ func (spc *realStatefulPodControl) createPersistentVolumeClaims(set *apps.Statef
 		case err != nil:
 			errs = append(errs, fmt.Errorf("failed to retrieve PVC %s: %s", claim.Name, err))
 			spc.recordClaimEvent("create", set, pod, &claim, err)
+		case err == nil:
+			if pvc.DeletionTimestamp != nil {
+				errs = append(errs, fmt.Errorf("pvc %s is being deleted", claim.Name))
+			}
 		}
 		// TODO: Check resource requirements and accessmodes, update if necessary
 	}

pkg/controller/statefulset/stateful_pod_control_test.go

@@ -20,6 +20,7 @@ import (
 	"errors"
 	"strings"
 	"testing"
+	"time"
 
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -29,6 +30,7 @@ import (
 	"k8s.io/client-go/tools/record"
 
 	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes/fake"
 	corelisters "k8s.io/client-go/listers/core/v1"
 	_ "k8s.io/kubernetes/pkg/apis/apps/install"
@@ -128,6 +130,42 @@ func TestStatefulPodControlCreatePodPvcCreateFailure(t *testing.T) {
 		}
 	}
 }
+func TestStatefulPodControlCreatePodPvcDeleting(t *testing.T) {
+	recorder := record.NewFakeRecorder(10)
+	set := newStatefulSet(3)
+	pod := newStatefulSetPod(set, 0)
+	fakeClient := &fake.Clientset{}
+	pvcs := getPersistentVolumeClaims(set, pod)
+	pvcIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
+	deleteTime := time.Date(2019, time.January, 1, 0, 0, 0, 0, time.UTC)
+	for k := range pvcs {
+		pvc := pvcs[k]
+		pvc.DeletionTimestamp = &metav1.Time{Time: deleteTime}
+		pvcIndexer.Add(&pvc)
+	}
+	pvcLister := corelisters.NewPersistentVolumeClaimLister(pvcIndexer)
+	control := NewRealStatefulPodControl(fakeClient, nil, nil, pvcLister, recorder)
+	fakeClient.AddReactor("create", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) {
+		create := action.(core.CreateAction)
+		return true, create.GetObject(), nil
+	})
+	fakeClient.AddReactor("create", "pods", func(action core.Action) (bool, runtime.Object, error) {
+		create := action.(core.CreateAction)
+		return true, create.GetObject(), nil
+	})
+	if err := control.CreateStatefulPod(set, pod); err == nil {
+		t.Error("Failed to produce error on deleting PVC")
+	}
+	events := collectEvents(recorder.Events)
+	if eventCount := len(events); eventCount != 1 {
+		t.Errorf("Deleting PVC: got %d events, but want 1", eventCount)
+	}
+	for i := range events {
+		if !strings.Contains(events[i], v1.EventTypeWarning) {
+			t.Errorf("Found unexpected non-warning event %s", events[i])
+		}
+	}
+}
 
 type fakeIndexer struct {
 	cache.Indexer