Fix: e2e Tests

Author: NishanthNalluri

cockroachdb/templates/crdb.yaml

@@ -70,8 +70,5 @@ spec:
       {{- if .Values.operator.service.ports.http.port }}
       hTTPPort: {{ .Values.operator.service.ports.http.port }}
       {{- end }}
-  # All properties below are solely to pass validation. They aren't used by the
-  # betaclusterctrl controller so the values don't matter so long as they're
-  # valid.
   dataStore: {}
 {{- end }}

cockroachdb/values.yaml

@@ -713,7 +713,7 @@ iap:
   # clientSecret:
 
 # Set the GODEBUG env var for the Go runtime. By default, disable Transparent Huge Pages,
-# which cause memory inefficiency for CRDB.
+# which cause memory inefficiency for CockroachDB.
 godebug:
   disablethp: "1"
 
@@ -730,11 +730,11 @@ operator:
   nameOverride: ""
   fullnameOverride: ""
 
-  # A map of CRDB cluster settings.
+  # A map of CockroachDB cluster settings.
   # See https://www.cockroachlabs.com/docs/stable/cluster-settings.html
   clusterSettings: ~
 
-    # Regions controls the number of CRDB nodes that are deployed per region.
+    # Regions controls the number of CockroachDB nodes that are deployed per region.
     # regions: ~
     # - code: us-central1
       # nodes: 3
@@ -759,7 +759,7 @@ operator:
       # cpu: 100m
       # memory: 128Mi
 
-  # dataStore specifies the disk configuration for the CRDB Node.
+  # dataStore specifies the disk configuration for the CockroachDB Node.
   dataStore:
     volumeClaimTemplate:
       metadata: {}
@@ -780,7 +780,7 @@ operator:
     # the number of days generated certs are valid for
     # validForDays: 3650
 
-    # External certificates for the CRDB cluster.
+    # External certificates for the CockroachDB cluster.
     externalCertificates:
       clientCaConfigMapName: ""
       nodeCaConfigMapName: ""
@@ -789,7 +789,7 @@ operator:
       nodeSecretName: ""
       rootSqlClientSecretName: ""
 
-  # RBAC settings for CRDB nodes
+  # RBAC settings for CockroachDB nodes
   rbac:
     # By default the service account will be the resource name. It will
     # be created during the installation along with a namespaced role and
@@ -823,7 +823,7 @@ operator:
       cloudProvider: k3d
       namespace: default
 
-  # PodLabels are the labels that should be applied to the underlying CRDB pod
+  # PodLabels are the labels that should be applied to the underlying CockroachDB pod
   podLabels:
     app.kubernetes.io/component: cockroachdb
 
@@ -832,14 +832,14 @@ operator:
     # Disable backup/restore to local disk by default.
     --external-io-dir: disabled
 
-  # Environment variables set on cockroachdb pods.
+  # Environment variables set on CockroachDB pods.
   env: []
 
   # Delay between cockroachdb pod restarts. Wait 3m by default to avoid
   # unavailability during restarts.
   rollingRestartDelay: 3m0s
 
-  # Topology spread constraints set on cockroachdb pods. Spread cockroachdb
+  # Topology spread constraints set on CockroachDB pods. Spread CockroachDB
   # pods across zones by default.
   topologySpreadConstraints:
     - maxSkew: 1

tests/e2e/install/cockroachdb_helm_e2e_test.go

@@ -50,8 +50,8 @@ func TestCockroachDbHelmInstall(t *testing.T) {
 	}
 
 	k8s.CreateNamespace(t, kubectlOptions, namespaceName)
-	// ... and make sure to delete the namespace at the end of the test
-	defer k8s.DeleteNamespace(t, kubectlOptions, namespaceName)
+	// ... and make sure to delete the namespace at the end of the test.
+	defer testutil.DeleteNamespace(t, k8sClient, namespaceName)
 
 	const testDBName = "testdb"
 
@@ -88,7 +88,7 @@ func TestCockroachDbHelmInstall(t *testing.T) {
 		}
 	}()
 
-	// Next we wait for the service endpoint
+	// Next we wait for the service endpoint.
 	serviceName := fmt.Sprintf("%s-cockroachdb-public", releaseName)
 	k8s.WaitUntilServiceAvailable(t, kubectlOptions, serviceName, 30, 2*time.Second)
 
@@ -123,8 +123,8 @@ func TestCockroachDbHelmInstallWithCAProvided(t *testing.T) {
 	}
 
 	k8s.CreateNamespace(t, kubectlOptions, namespaceName)
-	// ... and make sure to delete the namespace at the end of the test
-	defer k8s.DeleteNamespace(t, kubectlOptions, namespaceName)
+	// ... and make sure to delete the namespace at the end of the test.
+	defer testutil.DeleteNamespace(t, k8sClient, namespaceName)
 
 	certOutput, err := shell.RunCommandAndGetOutputE(t, cmd)
 	t.Log(certOutput)
@@ -168,7 +168,7 @@ func TestCockroachDbHelmInstallWithCAProvided(t *testing.T) {
 			},
 		)
 
-		// custom user CA certificate secret should not be deleted by pre-delete job
+		// custom user CA certificate secret should not be deleted by pre-delete job.
 		_, err = k8s.GetSecretE(t, kubectlOptions, crdbCluster.CaSecret)
 		require.NoError(t, err)
 	}()
@@ -180,7 +180,7 @@ func TestCockroachDbHelmInstallWithCAProvided(t *testing.T) {
 		}
 	}()
 
-	// Next we wait for the service endpoint
+	// Next we wait for the service endpoint.
 	serviceName := fmt.Sprintf("%s-cockroachdb-public", releaseName)
 	k8s.WaitUntilServiceAvailable(t, kubectlOptions, serviceName, 30, 2*time.Second)
 
@@ -190,7 +190,7 @@ func TestCockroachDbHelmInstallWithCAProvided(t *testing.T) {
 	testutil.RequireCRDBToFunction(t, crdbCluster, false)
 }
 
-// Test to check migration from Bring your own certificate method to self-sginer cert utility
+// Test to check migration from Bring your own certificate method to self-sginer cert utility.
 func TestCockroachDbHelmMigration(t *testing.T) {
 	namespaceName := "cockroach" + strings.ToLower(random.UniqueId())
 	kubectlOptions := k8s.NewKubectlOptions("", "", namespaceName)
@@ -250,8 +250,8 @@ func TestCockroachDbHelmMigration(t *testing.T) {
 	}
 
 	k8s.CreateNamespace(t, kubectlOptions, namespaceName)
-	// Make sure to delete the namespace at the end of the test
-	defer k8s.DeleteNamespace(t, kubectlOptions, namespaceName)
+	// Make sure to delete the namespace at the end of the test.
+	defer testutil.DeleteNamespace(t, k8sClient, namespaceName)
 
 	cmds := []shell.Command{cmdCa, cmdNode, cmdClient}
 	for i := range cmds {
@@ -270,7 +270,7 @@ func TestCockroachDbHelmMigration(t *testing.T) {
 		fmt.Sprintf("--from-file=%s/ca.crt", certsDir))
 	require.NoError(t, err)
 
-	// Setup the args
+	// Setup the args.
 	options := &helm.Options{
 		KubectlOptions: k8s.NewKubectlOptions("", "", namespaceName),
 		SetValues: patchHelmValues(map[string]string{
@@ -299,12 +299,12 @@ func TestCockroachDbHelmMigration(t *testing.T) {
 	testutil.RequireClusterToBeReadyEventuallyTimeout(t, crdbCluster, 600*time.Second)
 	time.Sleep(20 * time.Second)
 
-	// Setup the args for upgrade
+	// Setup the args for upgrade.
 	crdbCluster.NodeSecret = fmt.Sprintf("%s-cockroachdb-node-secret", releaseName)
 	crdbCluster.ClientSecret = fmt.Sprintf("%s-cockroachdb-client-secret", releaseName)
 	crdbCluster.CaSecret = fmt.Sprintf("%s-cockroachdb-ca-secret", releaseName)
 
-	// Default method is self-signer so no need to set explicitly
+	// Default method is self-signer so no need to set explicitly.
 	options = &helm.Options{
 		KubectlOptions: k8s.NewKubectlOptions("", "", namespaceName),
 		SetValues: patchHelmValues(map[string]string{
@@ -319,8 +319,8 @@ func TestCockroachDbHelmMigration(t *testing.T) {
 	}
 
 	// Upgrade the cockroachdb helm chart and checks installation should succeed.
-	// Upgrade is done in goRoutine to unblock the code flow
-	// While upgrading statefulset pods need to be deleted manually to consume the new certificate chain
+	// Upgrade is done in goRoutine to unblock the code flow.
+	// While upgrading statefulset pods need to be deleted manually to consume the new certificate chain.
 	wg := sync.WaitGroup{}
 	wg.Add(1)
 
@@ -338,7 +338,7 @@ func TestCockroachDbHelmMigration(t *testing.T) {
 
 	wg.Wait()
 
-	// Wait for the service endpoint
+	// Wait for the service endpoint.
 	k8s.WaitUntilServiceAvailable(t, kubectlOptions, publicService, 30, 2*time.Second)
 
 	testutil.RequireCertificatesToBeValid(t, crdbCluster)
@@ -360,7 +360,7 @@ func TestCockroachDbWithInsecureMode(t *testing.T) {
 
 	k8s.CreateNamespace(t, kubectlOptions, namespaceName)
 	// ... and make sure to delete the namespace at the end of the test
-	defer k8s.DeleteNamespace(t, kubectlOptions, namespaceName)
+	defer testutil.DeleteNamespace(t, k8sClient, namespaceName)
 
 	// Setup the args. For this test, we will set the following input values:
 	options := &helm.Options{
@@ -397,7 +397,7 @@ func TestCockroachDbWithCertManager(t *testing.T) {
 
 	k8s.CreateNamespace(t, kubectlOptions, namespaceName)
 	// ... and make sure to delete the namespace at the end of the test
-	defer k8s.DeleteNamespace(t, kubectlOptions, namespaceName)
+	defer testutil.DeleteNamespace(t, k8sClient, namespaceName)
 
 	certManagerHelmOptions := &helm.Options{
 		KubectlOptions: k8s.NewKubectlOptions("", "", "cert-manager"),
@@ -526,7 +526,7 @@ func testWALFailoverExistingCluster(t *testing.T, additionalValues map[string]st
 	}
 
 	k8s.CreateNamespace(t, kubectlOptions, namespaceName)
-	defer k8s.DeleteNamespace(t, kubectlOptions, namespaceName)
+	defer testutil.DeleteNamespace(t, k8sClient, namespaceName)
 
 	// Print the debug logs in case of test failure.
 	defer func() {

tests/k3d/dev-cluster.sh

@@ -26,7 +26,7 @@ REQUIRED_IMAGES=(
     "quay.io/jetstack/cert-manager-webhook:v1.11.0"
     "quay.io/jetstack/cert-manager-controller:v1.11.0"
     "quay.io/jetstack/cert-manager-ctl:v1.11.0"
-    "cockroachdb/cockroach:v24.3.6"
+    "cockroachdb/cockroach:v25.1.0"
     "${REGISTRY}/${REPOSITORY}:$(bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml)"
 )
 

tests/testutil/require.go

@@ -11,6 +11,9 @@ import (
 	"testing"
 	"time"
 
+	"github.com/cockroachdb/cockroach-operator/pkg/ptr"
+	v1 "k8s.io/api/rbac/v1"
+
 	"github.com/cockroachdb/cockroach-operator/pkg/database"
 	"github.com/cockroachdb/cockroach-operator/pkg/kube"
 	"github.com/gruntwork-io/terratest/modules/k8s"
@@ -37,6 +40,8 @@ type CockroachCluster struct {
 	IsCaUserProvided           bool
 }
 
+const role = "crdb-test-cockroachdb-node-reader"
+
 // RequireClusterToBeReadyEventuallyTimeout waits for all the CRDB pods to come into running state.
 func RequireClusterToBeReadyEventuallyTimeout(t *testing.T, crdbCluster CockroachCluster, timeout time.Duration) {
 
@@ -463,3 +468,52 @@ func WaitUntilPodDeleted(
 	}
 	log.Println(message)
 }
+
+// DeleteNamespace deletes a namespace by removing Finalizers and setting GracePeriodSeconds to 0 for immediate deletion.
+// Since we use force deletion, we don't need to wait for the namespace to be fully deleted.
+// Namespace will be cleaned up when cluster is deleted post e2e execution
+func DeleteNamespace(t *testing.T, k8sClient client.Client, namespace string) {
+	t.Helper()
+
+	crb := &v1.ClusterRoleBinding{}
+	if err := k8sClient.Get(context.Background(), types.NamespacedName{Name: role}, crb); err != nil {
+		t.Logf("Error getting ClusterRoleBinding %s: %v", role, err)
+	}
+
+	if err := k8sClient.Delete(context.Background(), crb); err != nil {
+		t.Logf("Error deleting ClusterRoleBinding %s: %v", role, err)
+	}
+	cr := &v1.ClusterRole{}
+	if err := k8sClient.Get(context.Background(), types.NamespacedName{Name: role}, cr); err != nil {
+		t.Logf("Error getting ClusterRole %s: %v", role, err)
+	}
+
+	if err := k8sClient.Delete(context.Background(), cr); err != nil {
+		t.Logf("Error deleting ClusterRole %s: %v", role, err)
+	}
+
+	ns := &corev1.Namespace{}
+	if err := k8sClient.Get(context.Background(), types.NamespacedName{Name: namespace}, ns); err != nil {
+		if apierrors.IsNotFound(err) {
+			return
+		}
+		t.Logf("Error getting namespace %s: %v", namespace, err)
+		return
+	}
+
+	if len(ns.Spec.Finalizers) > 0 {
+		ns.Spec.Finalizers = nil
+		if err := k8sClient.Update(context.Background(), ns); err != nil {
+			t.Logf("Error removing finalizers from namespace %s: %v", namespace, err)
+		}
+	}
+
+	deleteOptions := metav1.DeleteOptions{
+		GracePeriodSeconds: ptr.Int64(60),
+	}
+	if err := k8sClient.Delete(context.Background(), ns, &client.DeleteOptions{
+		Raw: &deleteOptions,
+	}); err != nil && !apierrors.IsNotFound(err) {
+		t.Logf("Error deleting namespace %s: %v", namespace, err)
+	}
+}