Merge #152692

152692: cloud: add fault injection test for gcs and azure r=jeffswenson a=jeffswenson

This adds fault injection tests for the Google and Azure blobstorage
clients. The logging injection hook was moved to a call path to ensure
it covers the GCP client.

Release note: none
Epic: [CRDB-53946](https://cockroachlabs.atlassian.net/browse/CRDB-53946)

Co-authored-by: Jeff Swenson <[email protected]>

Author: craig[bot]

pkg/cloud/azure/BUILD.bazel

@@ -66,6 +66,7 @@ go_test(
         "//pkg/testutils/skip",
         "//pkg/util/envutil",
         "//pkg/util/leaktest",
+        "//pkg/util/log",
         "@com_github_azure_azure_sdk_for_go_sdk_azcore//:azcore",
         "@com_github_azure_azure_sdk_for_go_sdk_azidentity//:azidentity",
         "@com_github_azure_azure_sdk_for_go_sdk_storage_azblob//service",

pkg/cloud/azure/azure_storage_test.go

@@ -13,8 +13,10 @@ import (
 	"os"
 	"path"
 	"testing"
+	"time"
 
 	"github.com/Azure/go-autorest/autorest/azure"
+	"github.com/cockroachdb/cockroach/pkg/base"
 	"github.com/cockroachdb/cockroach/pkg/cloud"
 	"github.com/cockroachdb/cockroach/pkg/cloud/cloudpb"
 	"github.com/cockroachdb/cockroach/pkg/cloud/cloudtestutils"
@@ -24,6 +26,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/testutils/skip"
 	"github.com/cockroachdb/cockroach/pkg/util/envutil"
 	"github.com/cockroachdb/cockroach/pkg/util/leaktest"
+	"github.com/cockroachdb/cockroach/pkg/util/log"
 	"github.com/cockroachdb/errors"
 	"github.com/stretchr/testify/require"
 )
@@ -151,6 +154,46 @@ func TestAzureSchemes(t *testing.T) {
 	}
 }
 
+func TestAzureFaultInjection(t *testing.T) {
+	defer leaktest.AfterTest(t)()
+
+	cfg, err := getAzureConfig()
+	if err != nil {
+		skip.IgnoreLint(t, "Test not configured for Azure")
+		return
+	}
+
+	// Enable cloud transport logging.
+	defer log.Scope(t).Close(t)
+	prevVModule := log.GetVModule()
+	defer func() { _ = log.SetVModule(prevVModule) }()
+	require.NoError(t, log.SetVModule("cloud_logging_transport=1"))
+
+	testID := cloudtestutils.NewTestID()
+	uri := cfg.filePathImplicitAuth(fmt.Sprintf("%d-fault-injection-test", testID))
+
+	// Inject faults for 15-45 seconds after the storage is opened.
+	middleware := cloudtestutils.BrownoutMiddleware(time.Second*15, time.Second*45)
+
+	conf, err := cloud.ExternalStorageConfFromURI(uri, username.RootUserName())
+	require.NoError(t, err)
+
+	args := cloud.EarlyBootExternalStorageContext{
+		IOConf:          base.ExternalIODirConfig{},
+		Settings:        cluster.MakeTestingClusterSettings(),
+		Options:         nil,
+		Limiters:        nil,
+		MetricsRecorder: cloud.NilMetrics,
+		HttpMiddleware:  middleware,
+	}
+
+	storage, err := makeAzureStorage(context.Background(), args, conf)
+	require.NoError(t, err)
+	defer storage.Close()
+
+	cloudtestutils.RunCloudNemesisTest(t, storage)
+}
+
 func TestAntagonisticAzureRead(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 

pkg/cloud/cloud_io.go

@@ -100,7 +100,7 @@ func MakeHTTPClient(
 	if err != nil {
 		return nil, err
 	}
-	return MakeHTTPClientForTransport(maybeAddLogging(t))
+	return MakeHTTPClientForTransport(t)
 }
 
 // MakeHTTPClientForTransport creates a new http.Client with the given
@@ -146,6 +146,7 @@ func MakeTransport(
 	if config.HttpMiddleware != nil {
 		roundTripper = config.HttpMiddleware(roundTripper)
 	}
+	roundTripper = maybeAddLogging(roundTripper)
 	return roundTripper, nil
 }
 

pkg/cloud/gcp/BUILD.bazel

@@ -60,6 +60,7 @@ go_test(
         "//pkg/testutils/skip",
         "//pkg/util/ioctx",
         "//pkg/util/leaktest",
+        "//pkg/util/log",
         "//pkg/util/uuid",
         "@com_github_cockroachdb_errors//:errors",
         "@com_github_googleapis_gax_go_v2//apierror",

pkg/cloud/gcp/gcs_storage_test.go

@@ -14,6 +14,7 @@ import (
 	"os"
 	"strings"
 	"testing"
+	"time"
 
 	gcs "cloud.google.com/go/storage"
 	"github.com/cockroachdb/cockroach/pkg/base"
@@ -25,6 +26,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/testutils/skip"
 	"github.com/cockroachdb/cockroach/pkg/util/ioctx"
 	"github.com/cockroachdb/cockroach/pkg/util/leaktest"
+	"github.com/cockroachdb/cockroach/pkg/util/log"
 	"github.com/cockroachdb/cockroach/pkg/util/uuid"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/oauth2/google"
@@ -297,6 +299,49 @@ func TestGCSAssumeRole(t *testing.T) {
 	})
 }
 
+func TestGCSFaultInjection(t *testing.T) {
+	defer leaktest.AfterTest(t)()
+
+	if !cloudtestutils.IsImplicitAuthConfigured() {
+		skip.IgnoreLint(t, "implicit auth is not configured")
+	}
+
+	bucket := os.Getenv("GOOGLE_BUCKET")
+	if bucket == "" {
+		skip.IgnoreLint(t, "GOOGLE_BUCKET env var must be set")
+	}
+
+	// Enable cloud transport logging.
+	defer log.Scope(t).Close(t)
+	prevVModule := log.GetVModule()
+	defer func() { _ = log.SetVModule(prevVModule) }()
+	require.NoError(t, log.SetVModule("cloud_logging_transport=1"))
+
+	testID := cloudtestutils.NewTestID()
+	uri := fmt.Sprintf("gs://%s/%d-fault-injection-test?AUTH=implicit", bucket, testID)
+
+	// Inject faults for 15-45 seconds after the storage is opened.
+	middleware := cloudtestutils.BrownoutMiddleware(time.Second*15, time.Second*45)
+
+	conf, err := cloud.ExternalStorageConfFromURI(uri, username.RootUserName())
+	require.NoError(t, err)
+
+	args := cloud.EarlyBootExternalStorageContext{
+		IOConf:          base.ExternalIODirConfig{},
+		Settings:        cluster.MakeTestingClusterSettings(),
+		Options:         nil,
+		Limiters:        nil,
+		MetricsRecorder: cloud.NilMetrics,
+		HttpMiddleware:  middleware,
+	}
+
+	storage, err := makeGCSStorage(context.Background(), args, conf)
+	require.NoError(t, err)
+	defer storage.Close()
+
+	cloudtestutils.RunCloudNemesisTest(t, storage)
+}
+
 func TestAntagonisticGCSRead(t *testing.T) {
 	defer leaktest.AfterTest(t)()