add ip discovery tests, extract deleteAll function to k8s_helpers

Author: nikitka

internal/controllers/database/controller_test.go

@@ -9,13 +9,16 @@ import (
 	"strings"
 	"testing"
 
+	"sigs.k8s.io/controller-runtime/pkg/envtest"
+
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 
@@ -30,12 +33,13 @@ import (
 var (
 	k8sClient client.Client
 	ctx       context.Context
+	env       *envtest.Environment
 )
 
 func TestAPIs(t *testing.T) {
 	RegisterFailHandler(Fail)
 
-	test.SetupK8STestManager(&ctx, &k8sClient, func(mgr *manager.Manager) []test.Reconciler {
+	env = test.SetupK8STestManager(&ctx, &k8sClient, func(mgr *manager.Manager) []test.Reconciler {
 		return []test.Reconciler{
 			&storage.Reconciler{
 				Client: k8sClient,
@@ -94,6 +98,7 @@ var _ = Describe("Database controller medium tests", func() {
 	AfterEach(func() {
 		Expect(k8sClient.Delete(ctx, &storageSample)).Should(Succeed())
 		Expect(k8sClient.Delete(ctx, &namespace)).Should(Succeed())
+		test.DeleteAllObjects(env, k8sClient, &namespace)
 	})
 
 	It("Checking field propagation to objects", func() {
@@ -217,4 +222,78 @@ var _ = Describe("Database controller medium tests", func() {
 			return reflect.DeepEqual(encryptionData, encryptionSecret.Data)
 		}, test.Timeout, test.Interval).Should(BeTrue())
 	})
+
+	It("Check iPDiscovery flag works", func() {
+		getDBSts := func(generation int64) appsv1.StatefulSet {
+			sts := appsv1.StatefulSet{}
+
+			Eventually(
+				func() error {
+					objectKey := types.NamespacedName{
+						Name:      testobjects.DatabaseName,
+						Namespace: testobjects.YdbNamespace,
+					}
+					err := k8sClient.Get(ctx, objectKey, &sts)
+					if err != nil {
+						return err
+					}
+
+					if sts.Generation <= generation {
+						return fmt.Errorf("sts is too old (generation=%d)", sts.Generation)
+					}
+					return nil
+				},
+			).WithTimeout(test.Timeout).WithPolling(test.Interval).Should(Succeed())
+
+			return sts
+		}
+		getDB := func() v1alpha1.Database {
+			found := v1alpha1.Database{}
+			Expect(k8sClient.Get(ctx, types.NamespacedName{
+				Name:      testobjects.DatabaseName,
+				Namespace: testobjects.YdbNamespace,
+			}, &found)).Should(Succeed())
+
+			return found
+		}
+
+		db := *testobjects.DefaultDatabase()
+
+		Expect(k8sClient.Create(ctx, &db)).Should(Succeed())
+
+		sts := getDBSts(0)
+		args := sts.Spec.Template.Spec.Containers[0].Args
+
+		Expect(args).To(ContainElements([]string{"--grpc-public-host"}))
+		Expect(args).ToNot(ContainElements([]string{"--grpc-public-address-v6", "--grpc-public-address-v4", "--grpc-public-target-name-override"}))
+
+		db = getDB()
+		db.Spec.Service.GRPC.IPDiscovery = &v1alpha1.IPDiscovery{
+			Enabled:  true,
+			IPFamily: corev1.IPv6Protocol,
+		}
+
+		Expect(k8sClient.Update(ctx, &db)).Should(Succeed())
+
+		sts = getDBSts(sts.Generation)
+		args = sts.Spec.Template.Spec.Containers[0].Args
+
+		Expect(args).To(ContainElements([]string{"--grpc-public-address-v6"}))
+		Expect(args).ToNot(ContainElements([]string{"--grpc-public-target-name-override"}))
+
+		db = getDB()
+
+		db.Spec.Service.GRPC.IPDiscovery = &v1alpha1.IPDiscovery{
+			Enabled:            true,
+			IPFamily:           corev1.IPv4Protocol,
+			TargetNameOverride: "a.b.c.d",
+		}
+
+		Expect(k8sClient.Update(ctx, &db)).Should(Succeed())
+
+		sts = getDBSts(sts.Generation)
+		args = sts.Spec.Template.Spec.Containers[0].Args
+
+		Expect(args).To(ContainElements([]string{"--grpc-public-address-v4", "--grpc-public-target-name-override"}))
+	})
 })

internal/controllers/remotedatabasenodeset/controller_test.go

@@ -14,10 +14,7 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
-	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
-	"k8s.io/client-go/kubernetes"
 	"k8s.io/kubectl/pkg/scheme"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/cache"
@@ -349,8 +346,8 @@ var _ = Describe("RemoteDatabaseNodeSet controller tests", func() {
 	AfterEach(func() {
 		Expect(localClient.Delete(ctx, databaseSample)).Should(Succeed())
 		Expect(localClient.Delete(ctx, storageSample)).Should(Succeed())
-		deleteAll(localEnv, localClient, &localNamespace)
-		deleteAll(remoteEnv, remoteClient, &localNamespace)
+		test.DeleteAllObjects(localEnv, localClient, &localNamespace)
+		test.DeleteAllObjects(remoteEnv, remoteClient, &localNamespace)
 	})
 
 	When("Created RemoteDatabaseNodeSet in k8s-mgmt-cluster", func() {
@@ -892,108 +889,3 @@ var _ = Describe("RemoteDatabaseNodeSet controller tests", func() {
 		})
 	})
 })
-
-func deleteAll(env *envtest.Environment, k8sClient client.Client, objs ...client.Object) {
-	for _, obj := range objs {
-		ctx := context.Background()
-		clientGo, err := kubernetes.NewForConfig(env.Config)
-		Expect(err).ShouldNot(HaveOccurred())
-		Expect(client.IgnoreNotFound(k8sClient.Delete(ctx, obj))).Should(Succeed())
-
-		//nolint:nestif
-		if ns, ok := obj.(*corev1.Namespace); ok {
-			// Normally the kube-controller-manager would handle finalization
-			// and garbage collection of namespaces, but with envtest, we aren't
-			// running a kube-controller-manager. Instead we're gonna approximate
-			// (poorly) the kube-controller-manager by explicitly deleting some
-			// resources within the namespace and then removing the `kubernetes`
-			// finalizer from the namespace resource so it can finish deleting.
-			// Note that any resources within the namespace that we don't
-			// successfully delete could reappear if the namespace is ever
-			// recreated with the same name.
-
-			// Look up all namespaced resources under the discovery API
-			_, apiResources, err := clientGo.Discovery().ServerGroupsAndResources()
-			Expect(err).ShouldNot(HaveOccurred())
-			namespacedGVKs := make(map[string]schema.GroupVersionKind)
-			for _, apiResourceList := range apiResources {
-				defaultGV, err := schema.ParseGroupVersion(apiResourceList.GroupVersion)
-				Expect(err).ShouldNot(HaveOccurred())
-				for _, r := range apiResourceList.APIResources {
-					if !r.Namespaced || strings.Contains(r.Name, "/") {
-						// skip non-namespaced and subresources
-						continue
-					}
-					gvk := schema.GroupVersionKind{
-						Group:   defaultGV.Group,
-						Version: defaultGV.Version,
-						Kind:    r.Kind,
-					}
-					if r.Group != "" {
-						gvk.Group = r.Group
-					}
-					if r.Version != "" {
-						gvk.Version = r.Version
-					}
-					namespacedGVKs[gvk.String()] = gvk
-				}
-			}
-
-			// Delete all namespaced resources in this namespace
-			for _, gvk := range namespacedGVKs {
-				var u unstructured.Unstructured
-				u.SetGroupVersionKind(gvk)
-				err := k8sClient.DeleteAllOf(ctx, &u, client.InNamespace(ns.Name))
-				Expect(client.IgnoreNotFound(ignoreMethodNotAllowed(err))).ShouldNot(HaveOccurred())
-			}
-
-			// Delete all Services in this namespace
-			serviceList := corev1.ServiceList{}
-			err = k8sClient.List(ctx, &serviceList, client.InNamespace(ns.Name))
-			Expect(err).ShouldNot(HaveOccurred())
-			for idx := range serviceList.Items {
-				policy := metav1.DeletePropagationForeground
-				err = k8sClient.Delete(ctx, &serviceList.Items[idx], &client.DeleteOptions{PropagationPolicy: &policy})
-				Expect(err).ShouldNot(HaveOccurred())
-			}
-
-			Eventually(func() error {
-				key := client.ObjectKeyFromObject(ns)
-				if err := k8sClient.Get(ctx, key, ns); err != nil {
-					return client.IgnoreNotFound(err)
-				}
-				// remove `kubernetes` finalizer
-				const kubernetes = "kubernetes"
-				finalizers := []corev1.FinalizerName{}
-				for _, f := range ns.Spec.Finalizers {
-					if f != kubernetes {
-						finalizers = append(finalizers, f)
-					}
-				}
-				ns.Spec.Finalizers = finalizers
-
-				// We have to use the k8s.io/client-go library here to expose
-				// ability to patch the /finalize subresource on the namespace
-				_, err = clientGo.CoreV1().Namespaces().Finalize(ctx, ns, metav1.UpdateOptions{})
-				return err
-			}, test.Timeout, test.Interval).Should(Succeed())
-		}
-
-		Eventually(func() metav1.StatusReason {
-			key := client.ObjectKeyFromObject(obj)
-			if err := k8sClient.Get(ctx, key, obj); err != nil {
-				return apierrors.ReasonForError(err)
-			}
-			return ""
-		}, test.Timeout, test.Interval).Should(Equal(metav1.StatusReasonNotFound))
-	}
-}
-
-func ignoreMethodNotAllowed(err error) error {
-	if err != nil {
-		if apierrors.ReasonForError(err) == metav1.StatusReasonMethodNotAllowed {
-			return nil
-		}
-	}
-	return err
-}

internal/controllers/remotestoragenodeset/controller_test.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"path/filepath"
 	"reflect"
-	"strings"
 	"testing"
 
 	. "github.com/onsi/ginkgo/v2"
@@ -14,10 +13,7 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
-	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
-	"k8s.io/client-go/kubernetes"
 	"k8s.io/kubectl/pkg/scheme"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/cache"
@@ -280,8 +276,8 @@ var _ = Describe("RemoteStorageNodeSet controller tests", func() {
 
 	AfterEach(func() {
 		Expect(localClient.Delete(ctx, storageSample)).Should(Succeed())
-		deleteAll(localEnv, localClient, &localNamespace)
-		deleteAll(remoteEnv, remoteClient, &localNamespace)
+		test.DeleteAllObjects(localEnv, localClient, &localNamespace)
+		test.DeleteAllObjects(remoteEnv, remoteClient, &localNamespace)
 	})
 
 	When("Created RemoteStorageNodeSet in k8s-mgmt-cluster", func() {
@@ -635,108 +631,3 @@ var _ = Describe("RemoteStorageNodeSet controller tests", func() {
 		})
 	})
 })
-
-func deleteAll(env *envtest.Environment, k8sClient client.Client, objs ...client.Object) {
-	for _, obj := range objs {
-		ctx := context.Background()
-		clientGo, err := kubernetes.NewForConfig(env.Config)
-		Expect(err).ShouldNot(HaveOccurred())
-		Expect(client.IgnoreNotFound(k8sClient.Delete(ctx, obj))).Should(Succeed())
-
-		//nolint:nestif
-		if ns, ok := obj.(*corev1.Namespace); ok {
-			// Normally the kube-controller-manager would handle finalization
-			// and garbage collection of namespaces, but with envtest, we aren't
-			// running a kube-controller-manager. Instead we're gonna approximate
-			// (poorly) the kube-controller-manager by explicitly deleting some
-			// resources within the namespace and then removing the `kubernetes`
-			// finalizer from the namespace resource so it can finish deleting.
-			// Note that any resources within the namespace that we don't
-			// successfully delete could reappear if the namespace is ever
-			// recreated with the same name.
-
-			// Look up all namespaced resources under the discovery API
-			_, apiResources, err := clientGo.Discovery().ServerGroupsAndResources()
-			Expect(err).ShouldNot(HaveOccurred())
-			namespacedGVKs := make(map[string]schema.GroupVersionKind)
-			for _, apiResourceList := range apiResources {
-				defaultGV, err := schema.ParseGroupVersion(apiResourceList.GroupVersion)
-				Expect(err).ShouldNot(HaveOccurred())
-				for _, r := range apiResourceList.APIResources {
-					if !r.Namespaced || strings.Contains(r.Name, "/") {
-						// skip non-namespaced and subresources
-						continue
-					}
-					gvk := schema.GroupVersionKind{
-						Group:   defaultGV.Group,
-						Version: defaultGV.Version,
-						Kind:    r.Kind,
-					}
-					if r.Group != "" {
-						gvk.Group = r.Group
-					}
-					if r.Version != "" {
-						gvk.Version = r.Version
-					}
-					namespacedGVKs[gvk.String()] = gvk
-				}
-			}
-
-			// Delete all namespaced resources in this namespace
-			for _, gvk := range namespacedGVKs {
-				var u unstructured.Unstructured
-				u.SetGroupVersionKind(gvk)
-				err := k8sClient.DeleteAllOf(ctx, &u, client.InNamespace(ns.Name))
-				Expect(client.IgnoreNotFound(ignoreMethodNotAllowed(err))).ShouldNot(HaveOccurred())
-			}
-
-			// Delete all Services in this namespace
-			serviceList := corev1.ServiceList{}
-			err = k8sClient.List(ctx, &serviceList, client.InNamespace(ns.Name))
-			Expect(err).ShouldNot(HaveOccurred())
-			for idx := range serviceList.Items {
-				policy := metav1.DeletePropagationForeground
-				err = k8sClient.Delete(ctx, &serviceList.Items[idx], &client.DeleteOptions{PropagationPolicy: &policy})
-				Expect(err).ShouldNot(HaveOccurred())
-			}
-
-			Eventually(func() error {
-				key := client.ObjectKeyFromObject(ns)
-				if err := k8sClient.Get(ctx, key, ns); err != nil {
-					return client.IgnoreNotFound(err)
-				}
-				// remove `kubernetes` finalizer
-				const kubernetes = "kubernetes"
-				finalizers := []corev1.FinalizerName{}
-				for _, f := range ns.Spec.Finalizers {
-					if f != kubernetes {
-						finalizers = append(finalizers, f)
-					}
-				}
-				ns.Spec.Finalizers = finalizers
-
-				// We have to use the k8s.io/client-go library here to expose
-				// ability to patch the /finalize subresource on the namespace
-				_, err = clientGo.CoreV1().Namespaces().Finalize(ctx, ns, metav1.UpdateOptions{})
-				return err
-			}, test.Timeout, test.Interval).Should(Succeed())
-		}
-
-		Eventually(func() metav1.StatusReason {
-			key := client.ObjectKeyFromObject(obj)
-			if err := k8sClient.Get(ctx, key, obj); err != nil {
-				return apierrors.ReasonForError(err)
-			}
-			return ""
-		}, test.Timeout, test.Interval).Should(Equal(metav1.StatusReasonNotFound))
-	}
-}
-
-func ignoreMethodNotAllowed(err error) error {
-	if err != nil {
-		if apierrors.ReasonForError(err) == metav1.StatusReasonMethodNotAllowed {
-			return nil
-		}
-	}
-	return err
-}