Merge cockroachdb#106668 cockroachdb#107205

106668: plpgsql: implement tail-call optimization for PLpgSQL routines r=DrewKimball a=DrewKimball

This patch implements tail-call optimization for the nested routine execution that is used to handle PLpgSQL control flow. PLpgSQL sub-routines are always tail calls because they are built as "continuation" functions, so we can always use the optimization for PLpgSQL. Tail-call optimization is only possible if the plan is not distributed (although we may not currently distribute such plans anyway).

The optimization is performed by setting a `deferredRoutineReceiver` field on the planner before planning and running a nested routine. This `deferredRoutineReceiver` allows a routine in tail-call position to send the information needed to evaluate itself to its parent, and then return NULL. Once the parent routine receives the result, it checks whether `deferredRoutineReceiver` received a deferred nested routine, and if so, evaluates it to obtain the actual result.

Given a simple looping function like the following:
```
CREATE FUNCTION f(n INT) RETURNS INT AS $$
  DECLARE
    i INT := 0;
  BEGIN
    LOOP
      IF i >= n THEN
        EXIT;
      END IF;
      i := i + 1;
    END LOOP;
    RETURN i;
  END
$$ LANGUAGE PLpgSQL;
```
This optimization takes runtime on my machine for `n=100000` from >20m to ~2s.

Informs cockroachdb#105254

Release note: None

107205: cloud: allow parallel running of cloud unit tests r=rhu713 a=rhu713

Append a random uint64 in the paths of cloud unit tests to prevent parallel executions from interfering with each other. This is necessary since these tests now run for all release branches and can run in parallel.

Fixes: cockroachdb#107137
Fixes: cockroachdb#107139

Release note: None

Co-authored-by: Drew Kimball <[email protected]>
Co-authored-by: Rui Hu <[email protected]>

Author: 3 people

pkg/cloud/amazon/s3_storage_test.go

@@ -79,9 +79,11 @@ func TestPutS3(t *testing.T) {
 
 	ctx := context.Background()
 	user := username.RootUserName()
+	testID := cloudtestutils.NewTestID()
+
 	t.Run("auth-empty-no-cred", func(t *testing.T) {
-		_, err := cloud.ExternalStorageFromURI(ctx, fmt.Sprintf("s3://%s/%s", bucket,
-			"backup-test-default"), base.ExternalIODirConfig{}, testSettings,
+		_, err := cloud.ExternalStorageFromURI(ctx, fmt.Sprintf("s3://%s/%s-%d", bucket,
+			"backup-test-default", testID), base.ExternalIODirConfig{}, testSettings,
 			blobs.TestEmptyBlobClientFactory, user,
 			nil, /* ie */
 			nil, /* ief */
@@ -109,15 +111,15 @@ func TestPutS3(t *testing.T) {
 		}
 
 		cloudtestutils.CheckExportStore(t, fmt.Sprintf(
-			"s3://%s/%s?%s=%s",
-			bucket, "backup-test-default",
+			"s3://%s/%s-%d?%s=%s",
+			bucket, "backup-test-default", testID,
 			cloud.AuthParam, cloud.AuthParamImplicit,
 		), false, user,
 			nil, /* db */
 			testSettings)
 	})
 	t.Run("auth-specified", func(t *testing.T) {
-		uri := S3URI(bucket, "backup-test",
+		uri := S3URI(bucket, fmt.Sprintf("backup-test-%d", testID),
 			&cloudpb.ExternalStorage_S3{AccessKey: creds.AccessKeyID, Secret: creds.SecretAccessKey, Region: "us-east-1"},
 		)
 		cloudtestutils.CheckExportStore(
@@ -142,8 +144,8 @@ func TestPutS3(t *testing.T) {
 		}
 
 		cloudtestutils.CheckExportStore(t, fmt.Sprintf(
-			"s3://%s/%s?%s=%s&%s=%s",
-			bucket, "backup-test-sse-256",
+			"s3://%s/%s-%d?%s=%s&%s=%s",
+			bucket, "backup-test-sse-256", testID,
 			cloud.AuthParam, cloud.AuthParamImplicit, AWSServerSideEncryptionMode,
 			"AES256",
 		),
@@ -158,8 +160,8 @@ func TestPutS3(t *testing.T) {
 			skip.IgnoreLint(t, "AWS_KMS_KEY_ARN env var must be set")
 		}
 		cloudtestutils.CheckExportStore(t, fmt.Sprintf(
-			"s3://%s/%s?%s=%s&%s=%s&%s=%s",
-			bucket, "backup-test-sse-kms",
+			"s3://%s/%s-%d?%s=%s&%s=%s&%s=%s",
+			bucket, "backup-test-sse-kms", testID,
 			cloud.AuthParam, cloud.AuthParamImplicit, AWSServerSideEncryptionMode,
 			"aws:kms", AWSServerSideEncryptionKMSID, v,
 		),
@@ -218,6 +220,8 @@ func TestPutS3AssumeRole(t *testing.T) {
 	}
 
 	testSettings := cluster.MakeTestingClusterSettings()
+	testID := cloudtestutils.NewTestID()
+	testPath := fmt.Sprintf("backup-test-%d", testID)
 
 	user := username.RootUserName()
 
@@ -232,7 +236,7 @@ func TestPutS3AssumeRole(t *testing.T) {
 			skip.IgnoreLintf(t, "we only run this test if a default role exists, "+
 				"refer to https://docs.aws.com/cli/latest/userguide/cli-configure-role.html: %s", err)
 		}
-		uri := S3URI(bucket, "backup-test",
+		uri := S3URI(bucket, testPath,
 			&cloudpb.ExternalStorage_S3{Auth: cloud.AuthParamImplicit, RoleARN: roleArn, Region: "us-east-1"},
 		)
 		cloudtestutils.CheckExportStore(
@@ -244,7 +248,7 @@ func TestPutS3AssumeRole(t *testing.T) {
 	})
 
 	t.Run("auth-specified", func(t *testing.T) {
-		uri := S3URI(bucket, "backup-test",
+		uri := S3URI(bucket, testPath,
 			&cloudpb.ExternalStorage_S3{Auth: cloud.AuthParamSpecified, RoleARN: roleArn, AccessKey: creds.AccessKeyID, Secret: creds.SecretAccessKey, Region: "us-east-1"},
 		)
 		cloudtestutils.CheckExportStore(
@@ -274,7 +278,7 @@ func TestPutS3AssumeRole(t *testing.T) {
 			t.Run(tc.auth, func(t *testing.T) {
 				// First verify that none of the individual roles in the chain can be used to access the storage.
 				for _, p := range providerChain {
-					roleURI := S3URI(bucket, "backup-test",
+					roleURI := S3URI(bucket, testPath,
 						&cloudpb.ExternalStorage_S3{
 							Auth:               tc.auth,
 							AssumeRoleProvider: p,
@@ -296,7 +300,7 @@ func TestPutS3AssumeRole(t *testing.T) {
 					delegatesWithoutID = append(delegatesWithoutID, cloudpb.ExternalStorage_AssumeRoleProvider{Role: p.Role})
 				}
 
-				uri := S3URI(bucket, "backup-test",
+				uri := S3URI(bucket, testPath,
 					&cloudpb.ExternalStorage_S3{
 						Auth:                  tc.auth,
 						AssumeRoleProvider:    roleWithoutID,
@@ -311,7 +315,7 @@ func TestPutS3AssumeRole(t *testing.T) {
 				)
 
 				// Finally, check that the chain of roles can be used to access the storage.
-				uri = S3URI(bucket, "backup-test",
+				uri = S3URI(bucket, testPath,
 					&cloudpb.ExternalStorage_S3{
 						Auth:                  tc.auth,
 						AssumeRoleProvider:    providerChain[len(providerChain)-1],
@@ -352,11 +356,12 @@ func TestPutS3Endpoint(t *testing.T) {
 		skip.IgnoreLint(t, "AWS_S3_BUCKET env var must be set")
 	}
 	user := username.RootUserName()
+	testID := cloudtestutils.NewTestID()
 
 	u := url.URL{
 		Scheme:   "s3",
 		Host:     bucket,
-		Path:     "backup-test",
+		Path:     fmt.Sprintf("backup-test-%d", testID),
 		RawQuery: q.Encode(),
 	}
 

pkg/cloud/azure/azure_storage_test.go

@@ -13,6 +13,7 @@ package azure
 import (
 	"context"
 	"encoding/base64"
+	"fmt"
 	"net/url"
 	"os"
 	"path"
@@ -109,12 +110,16 @@ func TestAzure(t *testing.T) {
 		return
 	}
 	testSettings := cluster.MakeTestingClusterSettings()
-	cloudtestutils.CheckExportStore(t, cfg.filePath("backup-test"),
+	testID := cloudtestutils.NewTestID()
+	testPath := fmt.Sprintf("backup-test-%d", testID)
+	testListPath := fmt.Sprintf("listing-test-%d", testID)
+
+	cloudtestutils.CheckExportStore(t, cfg.filePath(testPath),
 		false, username.RootUserName(),
 		nil, /* db */
 		testSettings,
 	)
-	cloudtestutils.CheckListFiles(t, cfg.filePath("listing-test"),
+	cloudtestutils.CheckListFiles(t, cfg.filePath(testListPath),
 		username.RootUserName(),
 		nil, /* db */
 		testSettings,
@@ -176,21 +181,23 @@ func TestAntagonisticAzureRead(t *testing.T) {
 		return
 	}
 	testSettings := cluster.MakeTestingClusterSettings()
+	testID := cloudtestutils.NewTestID()
+	antagonistPath := fmt.Sprintf("antagonistic-read-%d", testID)
 
 	conf, err := cloud.ExternalStorageConfFromURI(
-		cfg.filePath("antagonistic-read"), username.RootUserName())
+		cfg.filePath(antagonistPath), username.RootUserName())
 	require.NoError(t, err)
 
 	cloudtestutils.CheckAntagonisticRead(t, conf, testSettings)
 
 	clientAuthConf, err := cloud.ExternalStorageConfFromURI(
-		cfg.filePathClientAuth("antagonistic-read"), username.RootUserName())
+		cfg.filePathClientAuth(antagonistPath), username.RootUserName())
 	require.NoError(t, err)
 
 	cloudtestutils.CheckAntagonisticRead(t, clientAuthConf, testSettings)
 
 	implicitAuthConf, err := cloud.ExternalStorageConfFromURI(
-		cfg.filePathImplicitAuth("antagonistic-read"), username.RootUserName())
+		cfg.filePathImplicitAuth(antagonistPath), username.RootUserName())
 	require.NoError(t, err)
 
 	cloudtestutils.CheckAntagonisticRead(t, implicitAuthConf, testSettings)
@@ -277,11 +284,15 @@ func TestAzureStorageFileImplicitAuth(t *testing.T) {
 		return
 	}
 	testSettings := cluster.MakeTestingClusterSettings()
+	testID := cloudtestutils.NewTestID()
 
 	cleanup := envutil.TestSetEnv(t, "AZURE_CLIENT_ID", "")
 	defer cleanup()
 
-	cloudtestutils.CheckNoPermission(t, cfg.filePathImplicitAuth("backup-test"), username.RootUserName(),
+	testPath := fmt.Sprintf("backup-test-%d", testID)
+	testListPath := fmt.Sprintf("listing-test-%d", testID)
+
+	cloudtestutils.CheckNoPermission(t, cfg.filePathImplicitAuth(testPath), username.RootUserName(),
 		nil /*db*/, testSettings)
 
 	tmpDir, cleanup2 := testutils.TempDir(t)
@@ -293,12 +304,12 @@ func TestAzureStorageFileImplicitAuth(t *testing.T) {
 	cleanup3 := envutil.TestSetEnv(t, "COCKROACH_AZURE_APPLICATION_CREDENTIALS_FILE", credFile)
 	defer cleanup3()
 
-	cloudtestutils.CheckExportStore(t, cfg.filePathImplicitAuth("backup-test"),
+	cloudtestutils.CheckExportStore(t, cfg.filePathImplicitAuth(testPath),
 		false, username.RootUserName(),
 		nil, /* db */
 		testSettings,
 	)
-	cloudtestutils.CheckListFiles(t, cfg.filePathImplicitAuth("listing-test"),
+	cloudtestutils.CheckListFiles(t, cfg.filePathImplicitAuth(testListPath),
 		username.RootUserName(),
 		nil, /* db */
 		testSettings,

pkg/cloud/cloudtestutils/cloud_test_helpers.go

@@ -591,3 +591,8 @@ func IsImplicitAuthConfigured() bool {
 	credentials := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS")
 	return credentials != ""
 }
+
+func NewTestID() uint64 {
+	rng, _ := randutil.NewTestRand()
+	return rng.Uint64()
+}

pkg/cloud/gcp/gcs_storage_test.go

@@ -45,16 +45,18 @@ func TestPutGoogleCloud(t *testing.T) {
 
 	user := username.RootUserName()
 	testSettings := cluster.MakeTestingClusterSettings()
+	testID := cloudtestutils.NewTestID()
 
 	testutils.RunTrueAndFalse(t, "auth-specified-with-auth-param", func(t *testing.T, specified bool) {
 		credentials := os.Getenv("GOOGLE_CREDENTIALS_JSON")
 		if credentials == "" {
 			skip.IgnoreLint(t, "GOOGLE_CREDENTIALS_JSON env var must be set")
 		}
 		encoded := base64.StdEncoding.EncodeToString([]byte(credentials))
-		uri := fmt.Sprintf("gs://%s/%s?%s=%s",
+		uri := fmt.Sprintf("gs://%s/%s-%d?%s=%s",
 			bucket,
 			"backup-test-specified",
+			testID,
 			CredentialsParam,
 			url.QueryEscape(encoded),
 		)
@@ -69,9 +71,10 @@ func TestPutGoogleCloud(t *testing.T) {
 			nil, /* db */
 			testSettings,
 		)
-		cloudtestutils.CheckListFiles(t, fmt.Sprintf("gs://%s/%s/%s?%s=%s&%s=%s",
+		cloudtestutils.CheckListFiles(t, fmt.Sprintf("gs://%s/%s-%d/%s?%s=%s&%s=%s",
 			bucket,
 			"backup-test-specified",
+			testID,
 			"listing-test",
 			cloud.AuthParam,
 			cloud.AuthParamSpecified,
@@ -89,15 +92,16 @@ func TestPutGoogleCloud(t *testing.T) {
 
 		cloudtestutils.CheckExportStore(
 			t,
-			fmt.Sprintf("gs://%s/%s?%s=%s", bucket, "backup-test-implicit",
+			fmt.Sprintf("gs://%s/%s-%d?%s=%s", bucket, "backup-test-implicit", testID,
 				cloud.AuthParam, cloud.AuthParamImplicit),
 			false,
 			user,
 			nil, /* db */
 			testSettings)
-		cloudtestutils.CheckListFiles(t, fmt.Sprintf("gs://%s/%s/%s?%s=%s",
+		cloudtestutils.CheckListFiles(t, fmt.Sprintf("gs://%s/%s-%d/%s?%s=%s",
 			bucket,
 			"backup-test-implicit",
+			testID,
 			"listing-test",
 			cloud.AuthParam,
 			cloud.AuthParamImplicit,
@@ -121,9 +125,10 @@ func TestPutGoogleCloud(t *testing.T) {
 		token, err := ts.Token()
 		require.NoError(t, err, "getting token")
 
-		uri := fmt.Sprintf("gs://%s/%s?%s=%s",
+		uri := fmt.Sprintf("gs://%s/%s-%d?%s=%s",
 			bucket,
 			"backup-test-specified",
+			testID,
 			BearerTokenParam,
 			token.AccessToken,
 		)
@@ -135,9 +140,10 @@ func TestPutGoogleCloud(t *testing.T) {
 			user,
 			nil, /* db */
 			testSettings)
-		cloudtestutils.CheckListFiles(t, fmt.Sprintf("gs://%s/%s/%s?%s=%s&%s=%s",
+		cloudtestutils.CheckListFiles(t, fmt.Sprintf("gs://%s/%s-%d/%s?%s=%s&%s=%s",
 			bucket,
 			"backup-test-specified",
+			testID,
 			"listing-test",
 			cloud.AuthParam,
 			cloud.AuthParamSpecified,
@@ -163,6 +169,8 @@ func TestGCSAssumeRole(t *testing.T) {
 		skip.IgnoreLint(t, "ASSUME_SERVICE_ACCOUNT env var must be set")
 	}
 
+	testID := cloudtestutils.NewTestID()
+
 	t.Run("specified", func(t *testing.T) {
 		credentials := os.Getenv("GOOGLE_CREDENTIALS_JSON")
 		if credentials == "" {
@@ -172,17 +180,18 @@ func TestGCSAssumeRole(t *testing.T) {
 
 		// Verify that specified permissions with the credentials do not give us
 		// access to the bucket.
-		cloudtestutils.CheckNoPermission(t, fmt.Sprintf("gs://%s/%s?%s=%s", limitedBucket, "backup-test-assume-role",
+		cloudtestutils.CheckNoPermission(t, fmt.Sprintf("gs://%s/%s-%d?%s=%s", limitedBucket, "backup-test-assume-role", testID,
 			CredentialsParam, url.QueryEscape(encoded)), user,
 			nil, /* db */
 			testSettings,
 		)
 
 		cloudtestutils.CheckExportStore(
 			t,
-			fmt.Sprintf("gs://%s/%s?%s=%s&%s=%s&%s=%s",
+			fmt.Sprintf("gs://%s/%s-%d?%s=%s&%s=%s&%s=%s",
 				limitedBucket,
 				"backup-test-assume-role",
+				testID,
 				cloud.AuthParam,
 				cloud.AuthParamSpecified,
 				AssumeRoleParam,
@@ -192,9 +201,10 @@ func TestGCSAssumeRole(t *testing.T) {
 			nil, /* db */
 			testSettings,
 		)
-		cloudtestutils.CheckListFiles(t, fmt.Sprintf("gs://%s/%s/%s?%s=%s&%s=%s&%s=%s",
+		cloudtestutils.CheckListFiles(t, fmt.Sprintf("gs://%s/%s-%d/%s?%s=%s&%s=%s&%s=%s",
 			limitedBucket,
 			"backup-test-assume-role",
+			testID,
 			"listing-test",
 			cloud.AuthParam,
 			cloud.AuthParamSpecified,
@@ -215,20 +225,21 @@ func TestGCSAssumeRole(t *testing.T) {
 
 		// Verify that implicit permissions with the credentials do not give us
 		// access to the bucket.
-		cloudtestutils.CheckNoPermission(t, fmt.Sprintf("gs://%s/%s?%s=%s", limitedBucket, "backup-test-assume-role",
+		cloudtestutils.CheckNoPermission(t, fmt.Sprintf("gs://%s/%s-%d?%s=%s", limitedBucket, "backup-test-assume-role", testID,
 			cloud.AuthParam, cloud.AuthParamImplicit), user,
 			nil, /* db */
 			testSettings,
 		)
 
-		cloudtestutils.CheckExportStore(t, fmt.Sprintf("gs://%s/%s?%s=%s&%s=%s", limitedBucket, "backup-test-assume-role",
+		cloudtestutils.CheckExportStore(t, fmt.Sprintf("gs://%s/%s-%d?%s=%s&%s=%s", limitedBucket, "backup-test-assume-role", testID,
 			cloud.AuthParam, cloud.AuthParamImplicit, AssumeRoleParam, assumedAccount), false, user,
 			nil, /* db */
 			testSettings,
 		)
-		cloudtestutils.CheckListFiles(t, fmt.Sprintf("gs://%s/%s/%s?%s=%s&%s=%s",
+		cloudtestutils.CheckListFiles(t, fmt.Sprintf("gs://%s/%s-%d/%s?%s=%s&%s=%s",
 			limitedBucket,
 			"backup-test-assume-role",
+			testID,
 			"listing-test",
 			cloud.AuthParam,
 			cloud.AuthParamImplicit,
@@ -270,9 +281,10 @@ func TestGCSAssumeRole(t *testing.T) {
 				// to access the storage.
 				for _, role := range roleChain {
 					q.Set(AssumeRoleParam, role)
-					roleURI := fmt.Sprintf("gs://%s/%s/%s?%s",
+					roleURI := fmt.Sprintf("gs://%s/%s-%d/%s?%s",
 						limitedBucket,
 						"backup-test-assume-role",
+						testID,
 						"listing-test",
 						q.Encode(),
 					)
@@ -284,9 +296,10 @@ func TestGCSAssumeRole(t *testing.T) {
 
 				// Finally, check that the chain of roles can be used to access the storage.
 				q.Set(AssumeRoleParam, roleChainStr)
-				uri := fmt.Sprintf("gs://%s/%s/%s?%s",
+				uri := fmt.Sprintf("gs://%s/%s-%d/%s?%s",
 					limitedBucket,
 					"backup-test-assume-role",
+					testID,
 					"listing-test",
 					q.Encode(),
 				)
@@ -454,13 +467,14 @@ func TestReadFileAtReturnsSize(t *testing.T) {
 		skip.IgnoreLint(t, "GOOGLE_BUCKET env var must be set")
 	}
 
+	testID := cloudtestutils.NewTestID()
 	user := username.RootUserName()
 	ctx := context.Background()
 	testSettings := cluster.MakeTestingClusterSettings()
 	file := "testfile"
 	data := []byte("hello world")
 
-	gsURI := fmt.Sprintf("gs://%s/%s?AUTH=implicit", bucket, "read-file-at-returns-size")
+	gsURI := fmt.Sprintf("gs://%s/%s-%d?AUTH=implicit", bucket, "read-file-at-returns-size", testID)
 	conf, err := cloud.ExternalStorageConfFromURI(gsURI, user)
 	require.NoError(t, err)
 	args := cloud.ExternalStorageContext{