Add not-ready clusters E2E test

The e2e tests do not consider any unhealthy cluster. This commit adds a
test case where the cluster federates itself twice and makes one the
virtual federation not-ready by changing the API endpoint to an invalid
address. The ready cluster is labelled and a complete CRUD test
guarantees that the not-ready cluster does not impact operations if the
the cluster is not targeted.

Author: jonathanbeber

scripts/pre-commit.sh

@@ -105,6 +105,19 @@ function run-e2e-tests-with-in-memory-controllers() {
   ${IN_MEMORY_E2E_TEST_CMD}
 }
 
+function run-e2e-tests-with-not-ready-clusters() {
+  # Run the tests without any verbosity. The unhealthy nodes generate
+  # too much logs.
+  go test -timeout 900s ./test/e2e \
+    -args -kubeconfig=${HOME}/.kube/config \
+    -single-call-timeout=2m \
+    -ginkgo.randomizeAllSpecs \
+    -limited-scope=true \
+    -in-memory-controllers=true \
+    -simulate-federation=true \
+    -ginkgo.focus='\[NOT_READY\]'
+}
+
 function run-namespaced-e2e-tests() {
   local namespaced_e2e_test_cmd="${E2E_TEST_CMD} -kubefed-namespace=foo -limited-scope=true"
   # Run the placement test separately to avoid crud failures if
@@ -200,6 +213,9 @@ kubectl scale deployments kubefed-controller-manager -n kube-federation-system -
 echo "Running e2e tests with race detector against cluster-scoped kubefed with in-memory controllers"
 run-e2e-tests-with-in-memory-controllers
 
+echo "Running e2e tests with not-ready clusters"
+run-e2e-tests-with-not-ready-clusters
+
 # FederatedTypeConfig controller is needed to remove finalizers from
 # FederatedTypeConfigs in order to successfully delete the KubeFed
 # control plane in the next step.

test/common/crudtester.go

@@ -41,6 +41,7 @@ import (
 	"sigs.k8s.io/kubefed/pkg/apis/core/common"
 	"sigs.k8s.io/kubefed/pkg/apis/core/typeconfig"
 	fedv1a1 "sigs.k8s.io/kubefed/pkg/apis/core/v1alpha1"
+	"sigs.k8s.io/kubefed/pkg/apis/core/v1beta1"
 	genericclient "sigs.k8s.io/kubefed/pkg/client/generic"
 	"sigs.k8s.io/kubefed/pkg/controller/sync"
 	"sigs.k8s.io/kubefed/pkg/controller/sync/status"
@@ -65,6 +66,7 @@ type FederatedTypeCrudTester struct {
 	// operation that involves member clusters may take longer due to
 	// propagation latency.
 	clusterWaitTimeout time.Duration
+	clustersNamespace  string
 }
 
 type TestClusterConfig struct {
@@ -77,7 +79,7 @@ type TestCluster struct {
 	Client util.ResourceClient
 }
 
-func NewFederatedTypeCrudTester(testLogger TestLogger, typeConfig typeconfig.Interface, kubeConfig *rest.Config, testClusters map[string]TestCluster, waitInterval, clusterWaitTimeout time.Duration) (*FederatedTypeCrudTester, error) {
+func NewFederatedTypeCrudTester(testLogger TestLogger, typeConfig typeconfig.Interface, kubeConfig *rest.Config, testClusters map[string]TestCluster, clustersNamespace string, waitInterval, clusterWaitTimeout time.Duration) (*FederatedTypeCrudTester, error) {
 	return &FederatedTypeCrudTester{
 		tl:                 testLogger,
 		typeConfig:         typeConfig,
@@ -87,11 +89,12 @@ func NewFederatedTypeCrudTester(testLogger TestLogger, typeConfig typeconfig.Int
 		testClusters:       testClusters,
 		waitInterval:       waitInterval,
 		clusterWaitTimeout: clusterWaitTimeout,
+		clustersNamespace:  clustersNamespace,
 	}, nil
 }
 
-func (c *FederatedTypeCrudTester) CheckLifecycle(targetObject *unstructured.Unstructured, overrides []interface{}) {
-	fedObject := c.CheckCreate(targetObject, overrides)
+func (c *FederatedTypeCrudTester) CheckLifecycle(targetObject *unstructured.Unstructured, overrides []interface{}, selectors map[string]string) {
+	fedObject := c.CheckCreate(targetObject, overrides, selectors)
 
 	c.CheckStatusCreated(util.NewQualifiedName(fedObject))
 
@@ -104,7 +107,7 @@ func (c *FederatedTypeCrudTester) CheckLifecycle(targetObject *unstructured.Unst
 	c.CheckDelete(fedObject, false)
 }
 
-func (c *FederatedTypeCrudTester) Create(targetObject *unstructured.Unstructured, overrides []interface{}) *unstructured.Unstructured {
+func (c *FederatedTypeCrudTester) Create(targetObject *unstructured.Unstructured, overrides []interface{}, selectors map[string]string) *unstructured.Unstructured {
 	qualifiedName := util.NewQualifiedName(targetObject)
 	kind := c.typeConfig.GetTargetType().Kind
 	fedKind := c.typeConfig.GetFederatedType().Kind
@@ -113,10 +116,7 @@ func (c *FederatedTypeCrudTester) Create(targetObject *unstructured.Unstructured
 		c.tl.Fatalf("Error obtaining %s from %s %q: %v", fedKind, kind, qualifiedName, err)
 	}
 
-	fedObject, err = c.setAdditionalTestData(fedObject, overrides, targetObject.GetGenerateName())
-	if err != nil {
-		c.tl.Fatalf("Error setting overrides and placement on %s %q: %v", fedKind, qualifiedName, err)
-	}
+	fedObject = c.setAdditionalTestData(fedObject, overrides, selectors, targetObject.GetGenerateName())
 
 	return c.createResource(c.typeConfig.GetFederatedType(), fedObject)
 }
@@ -141,15 +141,15 @@ func (c *FederatedTypeCrudTester) resourceClient(apiResource metav1.APIResource)
 	return client
 }
 
-func (c *FederatedTypeCrudTester) CheckCreate(targetObject *unstructured.Unstructured, overrides []interface{}) *unstructured.Unstructured {
-	fedObject := c.Create(targetObject, overrides)
+func (c *FederatedTypeCrudTester) CheckCreate(targetObject *unstructured.Unstructured, overrides []interface{}, selectors map[string]string) *unstructured.Unstructured {
+	fedObject := c.Create(targetObject, overrides, selectors)
 
 	c.CheckPropagation(fedObject)
 	return fedObject
 }
 
 // AdditionalTestData additionally sets fixture overrides and placement clusternames into federated object
-func (c *FederatedTypeCrudTester) setAdditionalTestData(fedObject *unstructured.Unstructured, overrides []interface{}, generateName string) (*unstructured.Unstructured, error) {
+func (c *FederatedTypeCrudTester) setAdditionalTestData(fedObject *unstructured.Unstructured, overrides []interface{}, selectors map[string]string, generateName string) *unstructured.Unstructured {
 	fedKind := c.typeConfig.GetFederatedType().Kind
 	qualifiedName := util.NewQualifiedName(fedObject)
 
@@ -159,17 +159,23 @@ func (c *FederatedTypeCrudTester) setAdditionalTestData(fedObject *unstructured.
 			c.tl.Fatalf("Error updating overrides in %s %q: %v", fedKind, qualifiedName, err)
 		}
 	}
-	clusterNames := []string{}
-	for name := range c.testClusters {
-		clusterNames = append(clusterNames, name)
-	}
-	err := util.SetClusterNames(fedObject, clusterNames)
-	if err != nil {
-		c.tl.Fatalf("Error setting cluster names in %s %q: %v", fedKind, qualifiedName, err)
+	if selectors != nil {
+		if err := util.SetClusterSelector(fedObject, selectors); err != nil {
+			c.tl.Fatalf("Error setting cluster selectors for %s/%s: %v", fedObject.GetKind(), fedObject.GetName(), err)
+		}
+	} else {
+		clusterNames := []string{}
+		for name := range c.testClusters {
+			clusterNames = append(clusterNames, name)
+		}
+		err := util.SetClusterNames(fedObject, clusterNames)
+		if err != nil {
+			c.tl.Fatalf("Error setting cluster names in %s %q: %v", fedKind, qualifiedName, err)
+		}
 	}
 	fedObject.SetGenerateName(generateName)
 
-	return fedObject, err
+	return fedObject
 }
 
 func (c *FederatedTypeCrudTester) CheckUpdate(fedObject *unstructured.Unstructured) {
@@ -334,7 +340,14 @@ func (c *FederatedTypeCrudTester) CheckDelete(fedObject *unstructured.Unstructur
 	if deletingInCluster {
 		stateMsg = "not present"
 	}
+	clusters, err := util.ComputePlacement(fedObject, c.getClusters(), false)
+	if err != nil {
+		c.tl.Fatalf("Couldn't retrieve clusters for %s/%s: %v", federatedKind, name, err)
+	}
 	for clusterName, testCluster := range c.testClusters {
+		if !clusters.Has(clusterName) {
+			continue
+		}
 		namespace = util.QualifiedNameForCluster(clusterName, qualifiedName).Namespace
 		err = wait.PollImmediate(c.waitInterval, waitTimeout, func() (bool, error) {
 			obj, err := testCluster.Client.Resources(namespace).Get(context.Background(), name, metav1.GetOptions{})
@@ -425,16 +438,33 @@ func (c *FederatedTypeCrudTester) CheckReplicaSet(fedObject *unstructured.Unstru
 	}
 }
 
+func (c *FederatedTypeCrudTester) getClusters() []*v1beta1.KubeFedCluster {
+	client, err := genericclient.New(c.kubeConfig)
+	if err != nil {
+		c.tl.Fatalf("Failed to get kubefed clientset: %v", err)
+	}
+
+	fedClusters := []*v1beta1.KubeFedCluster{}
+	for cluster := range c.testClusters {
+		clusterResource := &v1beta1.KubeFedCluster{}
+		err = client.Get(context.Background(), clusterResource, c.clustersNamespace, cluster)
+		if err != nil {
+			c.tl.Fatalf("Cannot get cluster %s: %v", cluster, err)
+		}
+		fedClusters = append(fedClusters, clusterResource)
+	}
+	return fedClusters
+}
+
 // CheckPropagation checks propagation for the crud tester's clients
 func (c *FederatedTypeCrudTester) CheckPropagation(fedObject *unstructured.Unstructured) {
 	federatedKind := c.typeConfig.GetFederatedType().Kind
 	qualifiedName := util.NewQualifiedName(fedObject)
 
-	clusterNames, err := util.GetClusterNames(fedObject)
+	selectedClusters, err := util.ComputePlacement(fedObject, c.getClusters(), false)
 	if err != nil {
 		c.tl.Fatalf("Error retrieving cluster names for %s %q: %v", federatedKind, qualifiedName, err)
 	}
-	selectedClusters := sets.NewString(clusterNames...)
 
 	templateVersion, err := sync.GetTemplateHash(fedObject.Object)
 	if err != nil {

test/e2e/crd.go

@@ -215,10 +215,10 @@ overrides:
 		return targetObj, overrides, nil
 	}
 
-	crudTester, targetObject, overrides := initCrudTest(f, tl, typeConfig, testObjectsFunc)
+	crudTester, targetObject, overrides := initCrudTest(f, tl, f.KubeFedSystemNamespace(), typeConfig, testObjectsFunc)
 	// Make a copy for use in the orphan check.
 	deletionTargetObject := targetObject.DeepCopy()
-	crudTester.CheckLifecycle(targetObject, overrides)
+	crudTester.CheckLifecycle(targetObject, overrides, nil)
 
 	if namespaced {
 		// This check should not fail so long as the main test loop
@@ -228,7 +228,7 @@ overrides:
 			tl.Fatalf("Test of orphaned deletion assumes deletion of the containing namespace")
 		}
 		// Perform a check of orphan deletion.
-		fedObject := crudTester.CheckCreate(deletionTargetObject, nil)
+		fedObject := crudTester.CheckCreate(deletionTargetObject, nil, nil)
 		orphanDeletion := true
 		crudTester.CheckDelete(fedObject, orphanDeletion)
 	}

test/e2e/crud.go

@@ -61,8 +61,8 @@ var _ = Describe("Federated", func() {
 		Describe(fmt.Sprintf("%q", typeConfigName), func() {
 			It("should be created, read, updated and deleted successfully", func() {
 				typeConfig, testObjectsFunc := getCrudTestInput(f, tl, typeConfigName, fixture)
-				crudTester, targetObject, overrides := initCrudTest(f, tl, typeConfig, testObjectsFunc)
-				crudTester.CheckLifecycle(targetObject, overrides)
+				crudTester, targetObject, overrides := initCrudTest(f, tl, f.KubeFedSystemNamespace(), typeConfig, testObjectsFunc)
+				crudTester.CheckLifecycle(targetObject, overrides, nil)
 			})
 
 			for _, remoteStatusTypeName := range containedTypeNames {
@@ -78,8 +78,8 @@ var _ = Describe("Federated", func() {
 						tl.Logf("Show the content of the kubefedconfig file: '%v'", kubeFedConfig)
 
 						typeConfig, testObjectsFunc := getCrudTestInput(f, tl, typeConfigName, fixture)
-						crudTester, targetObject, overrides := initCrudTest(f, tl, typeConfig, testObjectsFunc)
-						fedObject := crudTester.CheckCreate(targetObject, overrides)
+						crudTester, targetObject, overrides := initCrudTest(f, tl, f.KubeFedSystemNamespace(), typeConfig, testObjectsFunc)
+						fedObject := crudTester.CheckCreate(targetObject, overrides, nil)
 
 						By("Checking the remote status filled for each federated resource for every cluster")
 						tl.Logf("Checking the existence of a remote status for each fedObj in every cluster: %v", fedObject)
@@ -105,12 +105,12 @@ var _ = Describe("Federated", func() {
 
 				typeConfig, testObjectsFunc := getCrudTestInput(f, tl, typeConfigName, fixture)
 				// Initialize the test without creating a federated namespace.
-				crudTester, targetObject, overrides := initCrudTestWithPropagation(f, tl, typeConfig, testObjectsFunc, false)
+				crudTester, targetObject, overrides := initCrudTestWithPropagation(f, tl, f.KubeFedSystemNamespace(), typeConfig, testObjectsFunc, false)
 
 				kind := typeConfig.GetFederatedType().Kind
 
 				By(fmt.Sprintf("Creating a %s whose containing namespace is not federated", kind))
-				fedObject := crudTester.Create(targetObject, overrides)
+				fedObject := crudTester.Create(targetObject, overrides, nil)
 
 				qualifiedName := util.NewQualifiedName(fedObject)
 
@@ -143,7 +143,7 @@ var _ = Describe("Federated", func() {
 
 			It("should have the managed label removed if not managed", func() {
 				typeConfig, testObjectsFunc := getCrudTestInput(f, tl, typeConfigName, fixture)
-				crudTester, targetObject, _ := initCrudTest(f, tl, typeConfig, testObjectsFunc)
+				crudTester, targetObject, _ := initCrudTest(f, tl, f.KubeFedSystemNamespace(), typeConfig, testObjectsFunc)
 
 				testClusters := crudTester.TestClusters()
 
@@ -192,7 +192,7 @@ var _ = Describe("Federated", func() {
 
 			It("should not be deleted if unlabeled", func() {
 				typeConfig, testObjectsFunc := getCrudTestInput(f, tl, typeConfigName, fixture)
-				crudTester, targetObject, _ := initCrudTest(f, tl, typeConfig, testObjectsFunc)
+				crudTester, targetObject, _ := initCrudTest(f, tl, f.KubeFedSystemNamespace(), typeConfig, testObjectsFunc)
 
 				testClusters := crudTester.TestClusters()
 
@@ -315,13 +315,13 @@ func getCrudTestInput(f framework.KubeFedFramework, tl common.TestLogger,
 	return typeConfig, testObjectsFunc
 }
 
-func initCrudTest(f framework.KubeFedFramework, tl common.TestLogger,
+func initCrudTest(f framework.KubeFedFramework, tl common.TestLogger, clustersNamespace string,
 	typeConfig typeconfig.Interface, testObjectsFunc testObjectsAccessor) (
 	*common.FederatedTypeCrudTester, *unstructured.Unstructured, []interface{}) {
-	return initCrudTestWithPropagation(f, tl, typeConfig, testObjectsFunc, true)
+	return initCrudTestWithPropagation(f, tl, clustersNamespace, typeConfig, testObjectsFunc, true)
 }
 
-func initCrudTestWithPropagation(f framework.KubeFedFramework, tl common.TestLogger,
+func initCrudTestWithPropagation(f framework.KubeFedFramework, tl common.TestLogger, clustersNamespace string,
 	typeConfig typeconfig.Interface, testObjectsFunc testObjectsAccessor,
 	ensureNamespacePropagation bool) (
 	*common.FederatedTypeCrudTester, *unstructured.Unstructured, []interface{}) {
@@ -343,7 +343,7 @@ func initCrudTestWithPropagation(f framework.KubeFedFramework, tl common.TestLog
 	targetAPIResource := typeConfig.GetTargetType()
 
 	testClusters := f.ClusterDynamicClients(&targetAPIResource, userAgent)
-	crudTester, err := common.NewFederatedTypeCrudTester(tl, typeConfig, kubeConfig, testClusters, framework.PollInterval, framework.TestContext.SingleCallTimeout)
+	crudTester, err := common.NewFederatedTypeCrudTester(tl, typeConfig, kubeConfig, testClusters, clustersNamespace, framework.PollInterval, framework.TestContext.SingleCallTimeout)
 	if err != nil {
 		tl.Fatalf("Error creating crudtester for %q: %v", federatedKind, err)
 	}

test/e2e/deleteoptions.go

@@ -39,8 +39,8 @@ var _ = Describe("DeleteOptions", func() {
 	It("Deployment should be created and deleted successfully, but ReplicaSet that created by Deployment won't be deleted", func() {
 
 		typeConfig, testObjectsFunc := getCrudTestInput(f, tl, typeConfigName, fixture)
-		crudTester, targetObject, overrides := initCrudTest(f, tl, typeConfig, testObjectsFunc)
-		fedObject := crudTester.CheckCreate(targetObject, overrides)
+		crudTester, targetObject, overrides := initCrudTest(f, tl, f.KubeFedSystemNamespace(), typeConfig, testObjectsFunc)
+		fedObject := crudTester.CheckCreate(targetObject, overrides, nil)
 
 		By("Set PropagationPolicy property as 'Orphan' on the DeleteOptions for Federated Deployment")
 		orphan := metav1.DeletePropagationOrphan

test/e2e/e2e.go

@@ -49,9 +49,9 @@ func RunE2ETests(t *testing.T) {
 var _ = ginkgo.SynchronizedBeforeSuite(func() []byte {
 	// Run only on Ginkgo node 1
 
-	if framework.TestContext.ScaleTest {
-		// Scale testing will initialize an in-memory control plane
-		// after the creation of a simulated federation.
+	// Some tests require simulated federation and will initialize an
+	// in-memory control plane.
+	if framework.TestContext.SimulateFederation {
 		return nil
 	}
 

test/e2e/framework/test_context.go

@@ -38,6 +38,7 @@ type TestContextType struct {
 	WaitForFinalization             bool
 	ScaleTest                       bool
 	ScaleClusterCount               int
+	SimulateFederation              bool
 }
 
 func (t *TestContextType) RunControllers() bool {
@@ -72,6 +73,7 @@ func registerFlags(t *TestContextType) {
 	flag.BoolVar(&t.WaitForFinalization, "wait-for-finalization", true,
 		"Whether the test suite should wait for finalization before stopping fixtures or exiting.  Setting this to false will speed up test execution but likely result in wedged namespaces and is only recommended for disposeable clusters.")
 	flag.BoolVar(&t.ScaleTest, "scale-test", false, "Whether the test suite should be configured for scale testing.  Not compatible with most tests.")
+	flag.BoolVar(&t.SimulateFederation, "simulate-federation", false, "Whether the tests require a simulated federation.")
 	flag.IntVar(&t.ScaleClusterCount, "scale-cluster-count", 1, "How many member clusters to simulate when scale testing.")
 }
 
@@ -83,6 +85,9 @@ func validateFlags(t *TestContextType) {
 	if t.ScaleTest {
 		t.InMemoryControllers = true
 		t.LimitedScope = true
+		// Scale testing will initialize an in-memory control plane
+		// after the creation of a simulated federation.
+		t.SimulateFederation = true
 		// Scale testing will create a namespace per simulated cluster
 		// and for large numbers of such namespaces the finalization
 		// wait could be considerable.