test(catalog): add registry pod delete e2e test

Author: njhale

test/e2e/catalog_e2e_test.go

@@ -184,14 +184,15 @@ func TestConfigMapUpdateTriggersRegistryPodRollout(t *testing.T) {
 	require.Equal(t, updatedConfigMap.GetResourceVersion(), fetchedUpdatedCatalog.Status.ConfigMapResource.ResourceVersion)
 
 	// Await 1 CatalogSource registry pod matching the updated labels
+	singlePod := podCount(1)
 	selector := labels.SelectorFromSet(map[string]string{"olm.catalogSource": mainCatalogName, "olm.configMapResourceVersion": updatedConfigMap.GetResourceVersion()})
-	podList, err := awaitPods(t, c, selector.String(), 1)
+	podList, err := awaitPods(t, c, testNamespace, selector.String(), singlePod)
 	require.NoError(t, err)
 	require.Equal(t, 1, len(podList.Items), "expected pod list not of length 1")
 
 	// Await 1 CatalogSource registry pod matching the updated labels
 	selector = labels.SelectorFromSet(map[string]string{"olm.catalogSource": mainCatalogName})
-	podList, err = awaitPods(t, c, selector.String(), 1)
+	podList, err = awaitPods(t, c, testNamespace, selector.String(), singlePod)
 	require.NoError(t, err)
 	require.Equal(t, 1, len(podList.Items), "expected pod list not of length 1")
 
@@ -377,9 +378,9 @@ func TestGrpcAddressCatalogSource(t *testing.T) {
 
 	// Replicate catalog pods with no OwnerReferences
 	mainCopy := replicateCatalogPod(t, c, crc, mainSource)
-	mainCopy = awaitPod(t, c, mainCopy.GetNamespace(), mainCopy.GetName(), HasPodIP)
+	mainCopy = awaitPod(t, c, mainCopy.GetNamespace(), mainCopy.GetName(), hasPodIP)
 	replacementCopy := replicateCatalogPod(t, c, crc, replacementSource)
-	replacementCopy = awaitPod(t, c, replacementCopy.GetNamespace(), replacementCopy.GetName(), HasPodIP)
+	replacementCopy = awaitPod(t, c, replacementCopy.GetNamespace(), replacementCopy.GetName(), hasPodIP)
 
 	addressSourceName := genName("address-catalog-")
 
@@ -435,6 +436,86 @@ func TestGrpcAddressCatalogSource(t *testing.T) {
 	require.NoError(t, err)
 }
 
+func TestDeleteRegistryPodTriggersRecreation(t *testing.T) {
+	// Create internal CatalogSource containing csv in package
+	// Wait for a registry pod to be created
+	// Create a Subscription for package
+	// Wait for the Subscription to succeed
+	// Wait for csv to succeed
+	// Delete the registry pod
+	// Wait for a new registry pod to be created
+
+	defer cleaner.NotifyTestComplete(t, true)
+
+	// Create internal CatalogSource containing csv in package
+	packageName := genName("nginx-")
+	packageStable := fmt.Sprintf("%s-stable", packageName)
+	stableChannel := "stable"
+	namedStrategy := newNginxInstallStrategy(genName("dep-"), nil, nil)
+	catalogName := genName("catsrc-")
+	crd := newCRD(genName("ins-"))
+	csv := newCSV(packageStable, testNamespace, "", *semver.New("0.1.0"), []apiextensions.CustomResourceDefinition{crd}, nil, namedStrategy)
+	manifests := []registry.PackageManifest{
+		{
+			PackageName: packageName,
+			Channels: []registry.PackageChannel{
+				{Name: stableChannel, CurrentCSVName: packageStable},
+			},
+			DefaultChannelName: stableChannel,
+		},
+	}
+
+	c := newKubeClient(t)
+	crc := newCRClient(t)
+	_, cleanupCatalog := createInternalCatalogSource(t, c, crc, catalogName, testNamespace, manifests, []apiextensions.CustomResourceDefinition{crd}, []v1alpha1.ClusterServiceVersion{csv})
+	defer cleanupCatalog()
+
+	// Wait for a new registry pod to be created
+	selector := labels.SelectorFromSet(map[string]string{"olm.catalogSource": catalogName})
+	singlePod := podCount(1)
+	registryPods, err := awaitPods(t, c, testNamespace, selector.String(), singlePod)
+	require.NoError(t, err, "error awaiting registry pod")
+	require.NotNil(t, registryPods, "nil registry pods")
+	require.Equal(t, 1, len(registryPods.Items), "unexpected number of registry pods found")
+
+	// Store the UID for later comparison
+	uid := registryPods.Items[0].GetUID()
+	name := registryPods.Items[0].GetName()
+
+	// Create a Subscription for package
+	subscriptionName := genName("sub-")
+	cleanupSubscription := createSubscriptionForCatalog(t, crc, testNamespace, subscriptionName, catalogName, packageName, stableChannel, "", v1alpha1.ApprovalAutomatic)
+	defer cleanupSubscription()
+
+	// Wait for the Subscription to succeed
+	subscription, err := fetchSubscription(t, crc, testNamespace, subscriptionName, subscriptionStateAtLatestChecker)
+	require.NoError(t, err)
+	require.NotNil(t, subscription)
+
+	// Wait for csv to succeed
+	_, err = fetchCSV(t, crc, subscription.Status.CurrentCSV, testNamespace, csvSucceededChecker)
+	require.NoError(t, err)
+
+	// Delete the registry pod
+	err = c.KubernetesInterface().CoreV1().Pods(testNamespace).Delete(name, &metav1.DeleteOptions{})
+	require.NoError(t, err)
+
+	// Wait for a new registry pod to be created
+	notUID := func(pods *corev1.PodList) bool {
+		for _, pod := range pods.Items {
+			if pod.GetUID() == uid {
+				return false
+			}
+		}
+
+		return true
+	}
+	registryPods, err = awaitPods(t, c, testNamespace, selector.String(), unionPodsCheck(singlePod, notUID))
+	require.NoError(t, err, "error waiting for replacement registry pod")
+	require.NotNil(t, registryPods, "nil replacement registry pods")
+	require.Equal(t, 1, len(registryPods.Items), "unexpected number of replacement registry pods found")
+}
+
 func getOperatorDeployment(c operatorclient.ClientInterface, namespace string, operatorLabels labels.Set) (*appsv1.Deployment, error) {
 	deployments, err := c.ListDeploymentsWithLabels(namespace, operatorLabels)
 	if err != nil || deployments == nil || len(deployments.Items) != 1 {

test/e2e/util_test.go

@@ -115,47 +115,96 @@ func newPMClient(t *testing.T) pmversioned.Interface {
 }
 
 // awaitPods waits for a set of pods to exist in the cluster
-func awaitPods(t *testing.T, c operatorclient.ClientInterface, selector string, expectedCount int) (*corev1.PodList, error) {
+func awaitPods(t *testing.T, c operatorclient.ClientInterface, namespace, selector string, checkPods podsCheckFunc) (*corev1.PodList, error) {
 	var fetchedPodList *corev1.PodList
 	var err error
 
 	err = wait.Poll(pollInterval, pollDuration, func() (bool, error) {
-		fetchedPodList, err = c.KubernetesInterface().CoreV1().Pods(testNamespace).List(metav1.ListOptions{
+		fetchedPodList, err = c.KubernetesInterface().CoreV1().Pods(namespace).List(metav1.ListOptions{
 			LabelSelector: selector,
 		})
 
 		if err != nil {
 			return false, err
 		}
 
-		t.Logf("Waiting for %d nodes matching %s selector, %d present", expectedCount, selector, len(fetchedPodList.Items))
+		t.Logf("Waiting for pods matching selector %s to match given conditions", selector)
 
-		if len(fetchedPodList.Items) < expectedCount {
-			return false, nil
-		}
-
-		return true, nil
+		return checkPods(fetchedPodList), nil
 	})
 
 	require.NoError(t, err)
 	return fetchedPodList, err
 }
 
-type PodCheckFunc func(pod *corev1.Pod) bool
+// podsCheckFunc describes a function that true if the given PodList meets some criteria; false otherwise.
+type podsCheckFunc func(pods *corev1.PodList) bool
+
+// unionPodsCheck returns a podsCheckFunc that represents the union of the given podsCheckFuncs.
+func unionPodsCheck(checks ...podsCheckFunc) podsCheckFunc {
+	return func(pods *corev1.PodList) bool {
+		for _, check := range checks {
+			if !check(pods) {
+				return false
+			}
+		}
+
+		return true
+	}
+}
+
+// podCount returns a podsCheckFunc that returns true if a PodList is of length count; false otherwise.
+func podCount(count int) podsCheckFunc {
+	return func(pods *corev1.PodList) bool {
+		return len(pods.Items) == count
+	}
+}
 
-func HasPodIP(pod *corev1.Pod) bool {
+// podsReady returns true if all of the pods in the given PodList have a ready condition with ConditionStatus "True"; false otherwise.
+func podsReady(pods *corev1.PodList) bool {
+	for _, pod := range pods.Items {
+		if !podReady(&pod) {
+			return false
+		}
+	}
+
+	return true
+}
+
+// podCheckFunc describes a function that returns true if the given Pod meets some criteria; false otherwise.
+type podCheckFunc func(pod *corev1.Pod) bool
+
+// hasPodIP returns true if the given Pod has a PodIP.
+func hasPodIP(pod *corev1.Pod) bool {
 	return pod.Status.PodIP != ""
 }
 
-func awaitPod(t *testing.T, c operatorclient.ClientInterface, namespace, name string, check PodCheckFunc) *corev1.Pod {
+// podReady returns true if the given Pod has a ready condition with ConditionStatus "True"; false otherwise.
+func podReady(pod *corev1.Pod) bool {
+	var status corev1.ConditionStatus
+	for _, condition := range pod.Status.Conditions {
+		if condition.Type != corev1.PodReady {
+			// Ignore all condition other than PodReady
+			continue
+		}
+
+		// Found PodReady condition
+		status = condition.Status
+		break
+	}
+
+	return status == corev1.ConditionTrue
+}
+
+func awaitPod(t *testing.T, c operatorclient.ClientInterface, namespace, name string, checkPod podCheckFunc) *corev1.Pod {
 	var pod *corev1.Pod
 	err := wait.Poll(pollInterval, pollDuration, func() (bool, error) {
 		p, err := c.KubernetesInterface().CoreV1().Pods(namespace).Get(name, metav1.GetOptions{})
 		if err != nil {
 			return false, err
 		}
 		pod = p
-		return check(pod), nil
+		return checkPod(pod), nil
 	})
 	require.NoError(t, err)