cloudccl: allow external connection tests to be run in parallel

Currently external connection tests read and write to the same path in cloud
storage. Add a random uint64 as part of the path so that test runs have unique
paths and can be run in parallel.

Fixes: cockroachdb#107407

Release note: None

Author: Rui Hu

pkg/ccl/cloudccl/amazon/BUILD.bazel

@@ -14,6 +14,7 @@ go_test(
         "//pkg/cloud",
         "//pkg/cloud/amazon",
         "//pkg/cloud/cloudpb",
+        "//pkg/cloud/cloudtestutils",
         "//pkg/cloud/externalconn/providers",
         "//pkg/security/securityassets",
         "//pkg/security/securitytest",

pkg/ccl/cloudccl/amazon/s3_connection_test.go

@@ -22,6 +22,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/cloud"
 	"github.com/cockroachdb/cockroach/pkg/cloud/amazon"
 	"github.com/cockroachdb/cockroach/pkg/cloud/cloudpb"
+	"github.com/cockroachdb/cockroach/pkg/cloud/cloudtestutils"
 	_ "github.com/cockroachdb/cockroach/pkg/cloud/externalconn/providers" // import External Connection providers.
 	"github.com/cockroachdb/cockroach/pkg/testutils"
 	"github.com/cockroachdb/cockroach/pkg/testutils/skip"
@@ -77,6 +78,8 @@ func TestS3ExternalConnection(t *testing.T) {
 		skip.IgnoreLint(t, "AWS_S3_BUCKET env var must be set")
 	}
 
+	testID := cloudtestutils.NewTestID()
+
 	t.Run("auth-implicit", func(t *testing.T) {
 		// You can create an IAM that can access S3
 		// in the AWS console, then set it up locally.
@@ -93,14 +96,14 @@ func TestS3ExternalConnection(t *testing.T) {
 		params := make(url.Values)
 		params.Add(cloud.AuthParam, cloud.AuthParamImplicit)
 
-		s3URI := fmt.Sprintf("s3://%s/backup-ec-test-default?%s", bucket, params.Encode())
+		s3URI := fmt.Sprintf("s3://%s/backup-ec-test-default-%d?%s", bucket, testID, params.Encode())
 		ecName := "auth-implicit-s3"
 		createExternalConnection(ecName, s3URI)
 		backupAndRestoreFromExternalConnection(ecName)
 	})
 
 	t.Run("auth-specified", func(t *testing.T) {
-		s3URI := amazon.S3URI(bucket, "backup-ec-test-default",
+		s3URI := amazon.S3URI(bucket, fmt.Sprintf("backup-ec-test-default-%d", testID),
 			&cloudpb.ExternalStorage_S3{
 				AccessKey: creds.AccessKeyID,
 				Secret:    creds.SecretAccessKey,
@@ -126,7 +129,7 @@ func TestS3ExternalConnection(t *testing.T) {
 				"refer to https://docs.aws.com/cli/latest/userguide/cli-configure-role.html: %s", err)
 		}
 
-		s3URI := amazon.S3URI(bucket, "backup-ec-test-sse-256", &cloudpb.ExternalStorage_S3{
+		s3URI := amazon.S3URI(bucket, fmt.Sprintf("backup-ec-test-sse-256-%d", testID), &cloudpb.ExternalStorage_S3{
 			Region:        "us-east-1",
 			Auth:          cloud.AuthParamImplicit,
 			ServerEncMode: "AES256",
@@ -139,7 +142,7 @@ func TestS3ExternalConnection(t *testing.T) {
 		if v == "" {
 			skip.IgnoreLint(t, "AWS_KMS_KEY_ARN env var must be set")
 		}
-		s3KMSURI := amazon.S3URI(bucket, "backup-ec-test-sse-kms", &cloudpb.ExternalStorage_S3{
+		s3KMSURI := amazon.S3URI(bucket, fmt.Sprintf("backup-ec-test-sse-kms-%d", testID), &cloudpb.ExternalStorage_S3{
 			Region:        "us-east-1",
 			Auth:          cloud.AuthParamImplicit,
 			ServerEncMode: "aws:kms",
@@ -163,7 +166,7 @@ func TestS3ExternalConnection(t *testing.T) {
 		}
 
 		// Unsupported server side encryption option.
-		invalidS3URI := amazon.S3URI(bucket, "backup-ec-test-sse-256", &cloudpb.ExternalStorage_S3{
+		invalidS3URI := amazon.S3URI(bucket, fmt.Sprintf("backup-ec-test-sse-256-%d", testID), &cloudpb.ExternalStorage_S3{
 			Region:        "us-east-1",
 			Auth:          cloud.AuthParamImplicit,
 			ServerEncMode: "unsupported-algorithm",
@@ -172,7 +175,7 @@ func TestS3ExternalConnection(t *testing.T) {
 			"unsupported server encryption mode unsupported-algorithm. Supported values are `aws:kms` and `AES256",
 			fmt.Sprintf(`BACKUP DATABASE foo INTO '%s'`, invalidS3URI))
 
-		invalidS3URI = amazon.S3URI(bucket, "backup-ec-test-sse-256", &cloudpb.ExternalStorage_S3{
+		invalidS3URI = amazon.S3URI(bucket, fmt.Sprintf("backup-ec-test-sse-256-%d", testID), &cloudpb.ExternalStorage_S3{
 			Region:        "us-east-1",
 			Auth:          cloud.AuthParamImplicit,
 			ServerEncMode: "aws:kms",
@@ -257,8 +260,9 @@ func TestAWSKMSExternalConnection(t *testing.T) {
 		skip.IgnoreLint(t, "AWS_S3_BUCKET env var must be set")
 	}
 
+	testID := cloudtestutils.NewTestID()
 	// Create an external connection where we will write the backup.
-	backupURI := fmt.Sprintf("s3://%s/backup?%s=%s", bucket,
+	backupURI := fmt.Sprintf("s3://%s/backup-%d?%s=%s", bucket, testID,
 		cloud.AuthParam, cloud.AuthParamImplicit)
 	backupExternalConnectionName := "backup"
 	createExternalConnection(backupExternalConnectionName, backupURI)
@@ -372,8 +376,9 @@ func TestAWSKMSExternalConnectionAssumeRole(t *testing.T) {
 		skip.IgnoreLint(t, "AWS_S3_BUCKET env var must be set")
 	}
 
+	testID := cloudtestutils.NewTestID()
 	// Create an external connection where we will write the backup.
-	backupURI := fmt.Sprintf("s3://%s/backup?%s=%s", bucket,
+	backupURI := fmt.Sprintf("s3://%s/backup-%d?%s=%s", bucket, testID,
 		cloud.AuthParam, cloud.AuthParamImplicit)
 	backupExternalConnectionName := "backup"
 	createExternalConnection(backupExternalConnectionName, backupURI)

pkg/ccl/cloudccl/azure/BUILD.bazel

@@ -13,6 +13,7 @@ go_test(
         "//pkg/ccl",
         "//pkg/ccl/kvccl/kvtenantccl",
         "//pkg/cloud/azure",
+        "//pkg/cloud/cloudtestutils",
         "//pkg/cloud/externalconn/providers",
         "//pkg/security/securityassets",
         "//pkg/security/securitytest",

pkg/ccl/cloudccl/azure/azure_connection_test.go

@@ -20,6 +20,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/base"
 	_ "github.com/cockroachdb/cockroach/pkg/ccl"
 	"github.com/cockroachdb/cockroach/pkg/cloud/azure"
+	"github.com/cockroachdb/cockroach/pkg/cloud/cloudtestutils"
 	_ "github.com/cockroachdb/cockroach/pkg/cloud/externalconn/providers" // import External Connection providers.
 	"github.com/cockroachdb/cockroach/pkg/testutils"
 	"github.com/cockroachdb/cockroach/pkg/testutils/skip"
@@ -29,9 +30,9 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/util/log"
 )
 
-func (a azureConfig) URI(file string) string {
-	return fmt.Sprintf("azure-storage://%s/%s?%s=%s&%s=%s&%s=%s",
-		a.bucket, file,
+func (a azureConfig) URI(file string, testID uint64) string {
+	return fmt.Sprintf("azure-storage://%s/%s-%d?%s=%s&%s=%s&%s=%s",
+		a.bucket, file, testID,
 		azure.AzureAccountKeyParam, url.QueryEscape(a.key),
 		azure.AzureAccountNameParam, url.QueryEscape(a.account),
 		azure.AzureEnvironmentKeyParam, url.QueryEscape(a.environment))
@@ -97,7 +98,8 @@ func TestExternalConnections(t *testing.T) {
 		return
 	}
 
+	testID := cloudtestutils.NewTestID()
 	ecName := "azure-ec"
-	createExternalConnection(ecName, cfg.URI("backup-ec"))
+	createExternalConnection(ecName, cfg.URI("backup-ec", testID))
 	backupAndRestoreFromExternalConnection(ecName)
 }

pkg/ccl/cloudccl/gcp/gcp_connection_test.go

@@ -89,8 +89,9 @@ func TestGCPKMSExternalConnection(t *testing.T) {
 		skip.IgnoreLint(t, "implicit auth is not configured")
 	}
 
+	testID := cloudtestutils.NewTestID()
 	// Create an external connection where we will write the backup.
-	backupURI := fmt.Sprintf("gs://%s/backup?%s=%s", bucket,
+	backupURI := fmt.Sprintf("gs://%s/backup-%d?%s=%s", bucket, testID,
 		cloud.AuthParam, cloud.AuthParamImplicit)
 	backupExternalConnectionName := "backup"
 	createExternalConnection(backupExternalConnectionName, backupURI)
@@ -229,8 +230,10 @@ func TestGCPKMSExternalConnectionAssumeRole(t *testing.T) {
 		skip.IgnoreLint(t, "implicit auth is not configured")
 	}
 
+	testID := cloudtestutils.NewTestID()
+
 	// Create an external connection where we will write the backup.
-	backupURI := fmt.Sprintf("gs://%s/backup?%s=%s", bucket,
+	backupURI := fmt.Sprintf("gs://%s/backup-%d?%s=%s", bucket, testID,
 		cloud.AuthParam, cloud.AuthParamImplicit)
 	backupExternalConnectionName := "backup"
 	createExternalConnection(backupExternalConnectionName, backupURI)
@@ -339,6 +342,7 @@ func TestGCPAssumeRoleExternalConnection(t *testing.T) {
 		skip.IgnoreLint(t, "ASSUME_SERVICE_ACCOUNT env var must be set")
 	}
 
+	testID := cloudtestutils.NewTestID()
 	t.Run("ec-assume-role-specified", func(t *testing.T) {
 		ecName := "ec-assume-role-specified"
 		disallowedECName := "ec-assume-role-specified-disallowed"
@@ -347,13 +351,14 @@ func TestGCPAssumeRoleExternalConnection(t *testing.T) {
 			skip.IgnoreLint(t, "GOOGLE_CREDENTIALS_JSON env var must be set")
 		}
 		encoded := base64.StdEncoding.EncodeToString([]byte(credentials))
-		disallowedURI := fmt.Sprintf("gs://%s/%s?%s=%s", limitedBucket, disallowedECName,
+		disallowedURI := fmt.Sprintf("gs://%s/%s-%d?%s=%s", limitedBucket, disallowedECName, testID,
 			gcp.CredentialsParam, url.QueryEscape(encoded))
 		disallowedCreateExternalConnection(t, disallowedECName, disallowedURI)
 
-		uri := fmt.Sprintf("gs://%s/%s?%s=%s&%s=%s&%s=%s",
+		uri := fmt.Sprintf("gs://%s/%s-%d?%s=%s&%s=%s&%s=%s",
 			limitedBucket,
 			ecName,
+			testID,
 			cloud.AuthParam,
 			cloud.AuthParamSpecified,
 			gcp.AssumeRoleParam,
@@ -370,13 +375,14 @@ func TestGCPAssumeRoleExternalConnection(t *testing.T) {
 		}
 		ecName := "ec-assume-role-implicit"
 		disallowedECName := "ec-assume-role-implicit-disallowed"
-		disallowedURI := fmt.Sprintf("gs://%s/%s?%s=%s", limitedBucket, disallowedECName,
+		disallowedURI := fmt.Sprintf("gs://%s/%s-%d?%s=%s", limitedBucket, disallowedECName, testID,
 			cloud.AuthParam, cloud.AuthParamImplicit)
 		disallowedCreateExternalConnection(t, disallowedECName, disallowedURI)
 
-		uri := fmt.Sprintf("gs://%s/%s?%s=%s&%s=%s",
+		uri := fmt.Sprintf("gs://%s/%s-%d?%s=%s&%s=%s",
 			limitedBucket,
 			ecName,
+			testID,
 			cloud.AuthParam,
 			cloud.AuthParamImplicit,
 			gcp.AssumeRoleParam,
@@ -418,17 +424,18 @@ func TestGCPAssumeRoleExternalConnection(t *testing.T) {
 					i := i
 					q.Set(gcp.AssumeRoleParam, role)
 					disallowedECName := fmt.Sprintf("ec-assume-role-checking-%d", i)
-					disallowedBackupURI := fmt.Sprintf("gs://%s/%s?%s", limitedBucket,
-						disallowedECName, q.Encode())
+					disallowedBackupURI := fmt.Sprintf("gs://%s/%s-%d?%s", limitedBucket,
+						disallowedECName, testID, q.Encode())
 					disallowedCreateExternalConnection(t, disallowedECName, disallowedBackupURI)
 				}
 
 				// Finally, check that the chain of roles can be used to access the storage.
 				q.Set(gcp.AssumeRoleParam, roleChainStr)
 				ecName := fmt.Sprintf("ec-assume-role-checking-%s", tc.auth)
-				uri := fmt.Sprintf("gs://%s/%s?%s",
+				uri := fmt.Sprintf("gs://%s/%s-%d?%s",
 					limitedBucket,
 					ecName,
+					testID,
 					q.Encode(),
 				)
 				createExternalConnection(t, ecName, uri)
@@ -477,13 +484,14 @@ func TestGCPExternalConnection(t *testing.T) {
 		skip.IgnoreLint(t, "GOOGLE_BUCKET env var must be set")
 	}
 
+	testID := cloudtestutils.NewTestID()
 	t.Run("ec-auth-implicit", func(t *testing.T) {
 		if !cloudtestutils.IsImplicitAuthConfigured() {
 			skip.IgnoreLint(t, "implicit auth is not configured")
 		}
 
 		ecName := "ec-auth-implicit"
-		backupURI := fmt.Sprintf("gs://%s/%s?%s=%s", bucket, ecName, cloud.AuthParam,
+		backupURI := fmt.Sprintf("gs://%s/%s-%d?%s=%s", bucket, ecName, testID, cloud.AuthParam,
 			cloud.AuthParamImplicit)
 		createExternalConnection(ecName, backupURI)
 		backupAndRestoreFromExternalConnection(ecName)
@@ -496,9 +504,10 @@ func TestGCPExternalConnection(t *testing.T) {
 		}
 		encoded := base64.StdEncoding.EncodeToString([]byte(credentials))
 		ecName := "ec-auth-specified"
-		backupURI := fmt.Sprintf("gs://%s/%s?%s=%s",
+		backupURI := fmt.Sprintf("gs://%s/%s-%d?%s=%s",
 			bucket,
 			ecName,
+			testID,
 			gcp.CredentialsParam,
 			url.QueryEscape(encoded),
 		)
@@ -520,9 +529,10 @@ func TestGCPExternalConnection(t *testing.T) {
 		token, err := ts.Token()
 		require.NoError(t, err, "getting token")
 		ecName := "ec-auth-specified-bearer-token"
-		backupURI := fmt.Sprintf("gs://%s/%s?%s=%s",
+		backupURI := fmt.Sprintf("gs://%s/%s-%d?%s=%s",
 			bucket,
 			ecName,
+			testID,
 			gcp.BearerTokenParam,
 			token.AccessToken,
 		)