WIP

Author: NishanthNalluri

Makefile

@@ -118,15 +118,11 @@ test/e2e/%: bin/cockroach bin/kubectl bin/helm build/self-signer test/cluster/up
 	$(MAKE) test/cluster/down; \
 	exit $${EXIT_CODE:-0}
 
-test/e2e/multi-region: bin/cockroach bin/kubectl bin/helm  build/self-signer test/single-cluster/up
-	@PATH="$(PWD)/bin:${PATH}" go test -timeout 60m -v -test.run TestOperatorInMultiRegion ./tests/e2e/operator/multiRegion/... || EXIT_CODE=$$?; \
-	$(MAKE) test/multi-cluster/down; \
-	exit $${EXIT_CODE:-0}
+test/e2e/multi-region: bin/cockroach bin/kubectl bin/helm  build/self-signer
+	@PATH="$(PWD)/bin:${PATH}" go test -timeout 60m -v -test.run TestOperatorInMultiRegion ./tests/e2e/operator/multiRegion/... || EXIT_CODE=$$?;
 
-test/e2e/single-region: bin/cockroach bin/kubectl bin/helm build/self-signer test/single-cluster/up
-	@PATH="$(PWD)/bin:${PATH}" go test -timeout 60m -v -test.run TestOperatorInSingleRegion ./tests/e2e/operator/singleRegion/... || EXIT_CODE=$$?; \
-	$(MAKE) test/multi-cluster/down; \
-	exit $${EXIT_CODE:-0}
+test/e2e/single-region: bin/cockroach bin/kubectl bin/helm build/self-signer
+	@PATH="$(PWD)/bin:${PATH}" go test -timeout 60m -v -test.run TestOperatorInSingleRegion ./tests/e2e/operator/singleRegion/... || EXIT_CODE=$$?;
 
 test/e2e/migrate: bin/cockroach bin/kubectl bin/helm bin/migration-helper build/self-signer test/cluster/up/3
 	@PATH="$(PWD)/bin:${PATH}" go test -timeout 30m -v ./tests/e2e/migrate/... || EXIT_CODE=$$?; \
@@ -137,7 +133,7 @@ test/single-cluster/up: bin/k3d
 	 ./tests/k3d/dev-multi-cluster.sh up --name "$(K3D_CLUSTER)" --nodes $(MULTI_REGION_NODE_SIZE) --clusters 1
 
 test/multi-cluster/down: bin/k3d
-	 ./tests/k3d/dev-multi-cluster.sh down --name "$(K3D_CLUSTER)" --nodes $(MULTI_REGION_NODE_SIZE) --clusters $(REGIONS)
+	 ./tests/k3d/dev-multi-cluster.sh down
 
 test/nightly-e2e/single-region: bin/cockroach bin/kubectl bin/helm build/self-signer
 	@PATH="$(PWD)/bin:${PATH}" go test -timeout 60m -v -test.run TestOperatorInSingleRegion ./tests/e2e/operator/singleRegion/... || EXIT_CODE=$$?; \

tests/e2e/operator/infra/common.go

@@ -1,7 +1,6 @@
 package infra
 
 import (
-	"context"
 	"fmt"
 	"net"
 	"os/exec"
@@ -56,11 +55,11 @@ const (
 
 // RegionCodes maps provider types to their region codes
 var RegionCodes = map[string][]string{
-	ProviderK3D:   {"us-east1", "us-east2", "us-west1"},
-	ProviderKind:  {"us-east1", "us-east2", "us-west1"},
+	ProviderK3D:   {"us-east1", "us-east2"},
+	ProviderKind:  {"us-east1", "us-east2"},
 	ProviderGCP:   {"us-central1", "us-east1", "us-west1"},
-	ProviderAzure: {"centralus", "eastus", "westus"},
-	ProviderAWS:   {"us-east-1", "us-east-2", "us-west-1"},
+	ProviderAzure: {"centralus", "eastus"},
+	ProviderAWS:   {"us-east-1", "us-east-2"},
 }
 
 // LoadBalancerAnnotations contains provider-specific service annotations
@@ -355,23 +354,3 @@ func UpdateKubeconfigAzure(t *testing.T, resourceGroup, clusterName string) erro
 	}
 	return nil
 }
-
-// WaitWithTimeout waits for a condition to be true with a timeout
-func WaitWithTimeout(ctx context.Context, interval, timeout time.Duration, condition func() (bool, error)) error {
-	deadline := time.Now().Add(timeout)
-	for time.Now().Before(deadline) {
-		success, err := condition()
-		if err != nil {
-			return err
-		}
-		if success {
-			return nil
-		}
-		select {
-		case <-ctx.Done():
-			return ctx.Err()
-		case <-time.After(interval):
-		}
-	}
-	return fmt.Errorf("timed out after %s", timeout)
-}

tests/e2e/operator/infra/gcp.go

@@ -33,7 +33,7 @@ const (
 	webhookFirewallRuleName  = "allow-9443-port-for-webhook"
 	internalFirewallRuleName = "allow-internal"
 	defaultNodePool          = "default-pool" // default node pool name for GKE clusters
-	subnetSuffix             = "subnet-1"     // suffix for dynamically created subnets
+	subnetSuffix             = "subnet"       // suffix for dynamically created subnets
 )
 
 // Helper functions to get network configuration from common.go
@@ -252,14 +252,60 @@ func (r *GcpRegion) TeardownInfra(t *testing.T) {
 	computeService, err := createComputeServiceClient(ctx)
 	require.NoError(t, err)
 
-	// 1) Delete GKE clusters via gcloud (ensures proper cleanup of node pools, etc.)
+	gkeService, err := createGKEServiceClient(ctx)
+	require.NoError(t, err)
+
+	// 1) Delete GKE clusters
 	for _, cfg := range clusterConfigurations[:len(r.Clusters)] {
 		t.Logf("[%s] Deleting GKE cluster '%s'", ProviderGCP, cfg.ClusterName)
+
+		// Check for ongoing operations
+		clusterPath := fmt.Sprintf("projects/%s/locations/%s/clusters/%s", projectID, cfg.Region, cfg.ClusterName)
+		cluster, err := gkeService.Projects.Locations.Clusters.Get(clusterPath).Context(ctx).Do()
+		if err != nil {
+			if IsResourceNotFound(err) {
+				t.Logf("[%s] Cluster %s already deleted", ProviderGCP, cfg.ClusterName)
+				continue
+			}
+			t.Logf("[%s] Warning: error checking cluster %s status: %v", ProviderGCP, cfg.ClusterName, err)
+			continue
+		}
+
+		// If cluster is in a transitioning state, wait for current operation to complete
+		if cluster.Status != "RUNNING" && cluster.Status != "ERROR" {
+			t.Logf("[%s] Cluster %s is in %s state, waiting for operation to complete...", ProviderGCP, cfg.ClusterName, cluster.Status)
+			if cluster.CurrentMasterVersion != "" { // Check if there's an ongoing operation
+				err = waitForGKEOperation(gkeService, cluster.CurrentMasterVersion, cfg.Region, "")
+				if err != nil {
+					t.Logf("[%s] Warning: error waiting for operation on cluster %s: %v", ProviderGCP, cfg.ClusterName, err)
+				}
+			}
+		}
+
+		// Now try to delete the cluster
 		delCmd := exec.Command("gcloud", "container", "clusters", "delete", cfg.ClusterName,
-			"--region", cfg.Region, "--project", projectID, "--quiet")
+			"--region", cfg.Region, "--project", projectID, "--quiet", "--async")
 		delCmd.Stdout = os.Stdout
 		delCmd.Stderr = os.Stderr
-		_ = delCmd.Run()
+		if err := delCmd.Run(); err != nil {
+			t.Logf("[%s] Warning: error initiating deletion of cluster %s: %v", ProviderGCP, cfg.ClusterName, err)
+		}
+	}
+
+	// Wait for all cluster deletions to complete
+	for _, cfg := range clusterConfigurations[:len(r.Clusters)] {
+		clusterPath := fmt.Sprintf("projects/%s/locations/%s/clusters/%s", projectID, cfg.Region, cfg.ClusterName)
+		for retries := 0; retries < 10; retries++ {
+			_, err := gkeService.Projects.Locations.Clusters.Get(clusterPath).Context(ctx).Do()
+			if IsResourceNotFound(err) {
+				t.Logf("[%s] Confirmed deletion of cluster %s", ProviderGCP, cfg.ClusterName)
+				break
+			}
+			if retries == 9 {
+				t.Logf("[%s] Warning: timed out waiting for cluster %s deletion", ProviderGCP, cfg.ClusterName)
+			}
+			time.Sleep(30 * time.Second)
+		}
 	}
 
 	// 2) Delete static IPs (unreserve)
@@ -296,22 +342,23 @@ func (r *GcpRegion) TeardownInfra(t *testing.T) {
 	t.Logf("[%s] Infrastructure teardown completed", ProviderGCP)
 }
 
+// This is a no-op right now
 func (r *GcpRegion) ScaleNodePool(t *testing.T, location string, nodeCount, index int) {
-	t.Logf("[%s] Scaling node pool for cluster '%s' to %d nodes", ProviderGCP, clusterConfigurations[index].ClusterName, nodeCount)
-
-	ctx := context.Background()
-	gkeService, err := createGKEServiceClient(ctx)
-	require.NoError(t, err, "failed to create GKE client")
-
-	scaleOp, err := scaleNodePool(ctx, gkeService, projectID, location, clusterConfigurations[index].ClusterName, defaultNodePool, int64(nodeCount))
-	require.NoError(t, err, "error initiating scaling for node pool")
-
-	err = waitForGKEOperation(gkeService, scaleOp.Name, location, "")
-	if err != nil {
-		t.Logf("[%s] Error during scaling operation for node pool '%s': %v", ProviderGCP, defaultNodePool, err)
-	} else {
-		t.Logf("[%s] Successfully scaled node pool '%s' to %d nodes", ProviderGCP, defaultNodePool, nodeCount)
-	}
+	//t.Logf("[%s] Scaling node pool for cluster '%s' to %d nodes", ProviderGCP, clusterConfigurations[index].ClusterName, nodeCount)
+	//
+	//ctx := context.Background()
+	//gkeService, err := createGKEServiceClient(ctx)
+	//require.NoError(t, err, "failed to create GKE client")
+	//
+	//scaleOp, err := scaleNodePool(ctx, gkeService, projectID, location, clusterConfigurations[index].ClusterName, defaultNodePool, int64(1))
+	//require.NoError(t, err, "error initiating scaling for node pool")
+	//
+	//err = waitForGKEOperation(gkeService, scaleOp.Name, location, "")
+	//if err != nil {
+	//	t.Logf("[%s] Error during scaling operation for node pool '%s': %v", ProviderGCP, defaultNodePool, err)
+	//} else {
+	//	t.Logf("[%s] Successfully scaled node pool '%s' to %d nodes", ProviderGCP, defaultNodePool, nodeCount)
+	//}
 }
 
 // getServiceAccountKeyPath returns the path to the service account key file
@@ -490,7 +537,10 @@ func createGKERegionalCluster(ctx context.Context, client *container.Service, se
 		"--tags", strings.Join([]string{defaultNodeTag}, ","), // Join tags if there are multiple
 		"--enable-master-authorized-networks",
 		"--master-authorized-networks", strings.Join([]string{"0.0.0.0/0"}, ","),
-		"--num-nodes", fmt.Sprint(DefaultNodesPerZone), // For regional, this is total nodes spread across 3 zones by default
+		"--num-nodes", fmt.Sprint(DefaultNodesPerZone),
+		"--min-nodes", fmt.Sprint(DefaultNodesPerZone),
+		"--max-nodes", fmt.Sprint(DefaultNodesPerZone + 1), // Needed for scaling cluster
+		"--enable-autoscaling", // Enable autoscaling
 		"--autoprovisioning-network-tags", strings.Join([]string{defaultNodeTag}, ","),
 		"--machine-type", GCPDefaultMachineType,
 		"--quiet", // Suppress interactive prompts

tests/e2e/operator/infra/k3d.go

@@ -7,12 +7,11 @@ import (
 	"testing"
 	"time"
 
-	"github.com/gruntwork-io/terratest/modules/random"
-
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/helm-charts/tests/e2e/calico"
 	"github.com/cockroachdb/helm-charts/tests/e2e/coredns"
 	"github.com/cockroachdb/helm-charts/tests/e2e/operator"
+	"github.com/cockroachdb/helm-charts/tests/testutil"
 	"github.com/gruntwork-io/terratest/modules/k8s"
 	"github.com/gruntwork-io/terratest/modules/retry"
 	"github.com/gruntwork-io/terratest/modules/shell"
@@ -32,7 +31,23 @@ type K3dRegion struct {
 
 // TeardownInfra cleans up all resources created by SetUpInfra
 func (r *K3dRegion) TeardownInfra(t *testing.T) {
-	t.Logf("[%s] K3D teardown not implemented - clusters will be cleaned up by the test framework", ProviderK3D)
+	t.Logf("[%s] Tearing down K3D infrastructure", ProviderK3D)
+
+	cmd := shell.Command{
+		Command: "make",
+		Args: []string{
+			"test/multi-cluster/down",
+		},
+		WorkingDir: testutil.GetGitRoot(),
+	}
+
+	output, err := shell.RunCommandAndGetOutputE(t, cmd)
+	if err != nil {
+		t.Logf("[%s] Warning: Failed to tear down K3D clusters: %v\nOutput: %s",
+			ProviderK3D, err, output)
+	} else {
+		t.Logf("[%s] Successfully tore down K3D clusters", ProviderK3D)
+	}
 }
 
 // ScaleNodePool scales the node pool in a K3D cluster
@@ -59,11 +74,10 @@ func (r *K3dRegion) SetUpInfra(t *testing.T) {
 	for i, cluster := range r.Clusters {
 		if _, ok := rawConfig.Contexts[cluster]; !ok {
 			// Create cluster using shell command.
-			err := createK3DCluster(t)
+			err := createK3DCluster(t, cluster, r.NodeCount)
 			require.NoError(t, err)
 		}
 
-		r.Namespace[cluster] = fmt.Sprintf("%s-%s", operator.Namespace, strings.ToLower(random.UniqueId()))
 		cfg, err := config.GetConfigWithContext(cluster)
 		require.NoError(t, err)
 		k8sClient, err := client.New(cfg, client.Options{})
@@ -212,14 +226,16 @@ func (r *K3dRegion) setupNetworking(t *testing.T, ctx context.Context, region st
 // createK3DCluster creates a new k3d cluster
 // by calling the make command which will create
 // single k3d cluster.
-func createK3DCluster(t *testing.T) error {
-	t.Logf("[%s] Creating new K3D cluster", ProviderK3D)
+func createK3DCluster(t *testing.T, clusterName string, nodeCount int) error {
+	t.Logf("[%s] Creating new K3D cluster: %s with %d nodes", ProviderK3D, clusterName, nodeCount)
 	cmd := shell.Command{
 		Command: "make",
 		Args: []string{
 			"test/single-cluster/up",
+			fmt.Sprintf("name=%s", strings.TrimLeft(clusterName, "k3d-")),
+			fmt.Sprintf("nodes=%d", nodeCount),
 		},
-		WorkingDir: "../../../../..",
+		WorkingDir: testutil.GetGitRoot(),
 	}
 
 	output, err := shell.RunCommandAndGetOutputE(t, cmd)
@@ -228,6 +244,6 @@ func createK3DCluster(t *testing.T) error {
 		return fmt.Errorf("failed to create cluster: %v\nOutput: %s", err, output)
 	}
 
-	t.Logf("[%s] Successfully created new K3D cluster", ProviderK3D)
+	t.Logf("[%s] Successfully created new K3D cluster: %s", ProviderK3D, clusterName)
 	return nil
 }

tests/e2e/operator/infra/provider.go

@@ -7,22 +7,19 @@ import (
 )
 
 // CloudProvider defines the interface that all cloud providers must implement
+// Some methods are optional - providers that don't support certain operations
+// can implement them as no-ops with appropriate logging
 type CloudProvider interface {
 	// SetUpInfra creates the necessary infrastructure for the tests
+	// This is the only required method for all providers
 	SetUpInfra(t *testing.T)
-}
 
-// CloudProviderWithTeardown extends CloudProvider with teardown capability
-type CloudProviderWithTeardown interface {
-	CloudProvider
 	// TeardownInfra cleans up all resources created by SetUpInfra
+	// Optional: providers that don't support teardown can implement as no-op
 	TeardownInfra(t *testing.T)
-}
 
-// CloudProviderWithScaling extends CloudProvider with scaling capability
-type CloudProviderWithScaling interface {
-	CloudProvider
 	// ScaleNodePool scales the node pool in a cluster
+	// Optional: providers that don't support scaling can implement as no-op
 	ScaleNodePool(t *testing.T, location string, nodeCount, index int)
 }
 
@@ -55,17 +52,27 @@ func ProviderFactory(providerType string, region *operator.Region) CloudProvider
 }
 
 // CanTeardown checks if the provider supports teardown
-func CanTeardown(provider CloudProvider) (CloudProviderWithTeardown, bool) {
-	if p, ok := provider.(CloudProviderWithTeardown); ok {
-		return p, true
+// This function is kept for backward compatibility
+func CanTeardown(provider CloudProvider) (CloudProvider, bool) {
+	// Check if the TeardownInfra method is a no-op implementation
+	// Kind and K3D providers have no-op implementations
+	switch provider.(type) {
+	case *KindRegion:
+		return nil, false
+	default:
+		return provider, true
 	}
-	return nil, false
 }
 
 // CanScale checks if the provider supports scaling
-func CanScale(provider CloudProvider) (CloudProviderWithScaling, bool) {
-	if p, ok := provider.(CloudProviderWithScaling); ok {
-		return p, true
+// This function is kept for backward compatibility
+func CanScale(provider CloudProvider) (CloudProvider, bool) {
+	// Check if the ScaleNodePool method is a no-op implementation
+	// GCP, Kind and K3D providers have no-op implementations
+	switch provider.(type) {
+	case *K3dRegion, *KindRegion, *GcpRegion:
+		return nil, false
+	default:
+		return provider, true
 	}
-	return nil, false
 }

tests/e2e/operator/multiRegion/cockroachdb_multi_region_e2e_test.go

@@ -52,7 +52,9 @@ func TestOperatorInMultiRegion(t *testing.T) {
 		provider := provider // Create new variable to avoid closure issues
 		t.Run(provider, func(t *testing.T) {
 			r.Provider = provider
-			r.Clusters = append(r.Clusters, fmt.Sprintf("%s-%s", r.Provider, operator.Clusters[0]))
+			for _, cluster := range operator.Clusters {
+				r.Clusters = append(r.Clusters, fmt.Sprintf("%s-%s", r.Provider, cluster))
+			}
 
 			t.Run("TestHelmInstall", r.TestHelmInstall)
 			t.Run("TestHelmUpgrade", r.TestHelmUpgrade)
@@ -374,7 +376,7 @@ func (r *multiRegion) TestClusterScaleUp(t *testing.T) {
 	// Modify the nodes in each region and apply helm upgrade.
 	for i, cluster := range r.Clusters {
 		kubectlOptions := k8s.NewKubectlOptions(cluster, kubeConfig, r.Namespace[cluster])
-		r.NodeCount = 4
+		r.NodeCount += 1
 
 		// Scale the node pool in the cloud infrastructure
 		r.scaleNodePool(t, r.RegionCodes[i], r.NodeCount, i)
@@ -395,10 +397,6 @@ func (r *multiRegion) TestClusterScaleUp(t *testing.T) {
 			DesiredNodes: r.NodeCount,
 		}
 		testutil.RequireCRDBClusterToBeReadyEventuallyTimeout(t, kubectlOptions, crdbCluster, 600*time.Second)
-		pods := k8s.ListPods(t, kubectlOptions, metav1.ListOptions{
-			LabelSelector: operator.LabelSelector,
-		})
-		require.True(t, len(pods) == 4)
 		// Validate CockroachDB cluster.
 		r.ValidateCRDB(t, cluster)
 	}