Increase HTTP timeout in tests to 5m (#4597)

## Changes
Currently the TF provider uses the Go SDK's default HTTP timeout of 60
seconds to wait for a response from the API. This is coupled with a 60
second server-side timeout for the API proxy. However, backend services
that expect to take longer to handle requests can have an increased
server-side timeout. In this case, the client may need to wait for
longer than 60 seconds to receive a response. As a result, some
integration tests for the provider fail with errors like "request timed
out after 1m0s".

This PR adjusts this by configuring the SDK used by the provider to have
a 10-minute timeout during integration tests. This is longer than any
individual server's server-side timeout. Long-term, the SDK should be
responsible for selecting the appropriate client-side timeout for a
given API, but in the meantime we'll make this adjustment.

This only affects tests. The default value for http_timeout_seconds is
still 60 seconds.

To-do:
- [ ] Add doccomments to new exported methods/functions/types.

## Tests
- [ ] Run integration tests, see that they pass.

NO_CHANGELOG=true

Author: mgyucht

internal/acceptance/init.go

@@ -22,9 +22,13 @@ import (
 	"github.com/databricks/terraform-provider-databricks/commands"
 	"github.com/databricks/terraform-provider-databricks/common"
 	"github.com/databricks/terraform-provider-databricks/internal/providers"
+	"github.com/databricks/terraform-provider-databricks/internal/providers/pluginfw"
+	"github.com/databricks/terraform-provider-databricks/internal/providers/sdkv2"
 	dbproviderlogger "github.com/databricks/terraform-provider-databricks/logger"
 	"github.com/databricks/terraform-provider-databricks/qa"
+	"github.com/hashicorp/terraform-plugin-framework/provider"
 	"github.com/hashicorp/terraform-plugin-go/tfprotov6"
+	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
 	"github.com/hashicorp/terraform-plugin-testing/helper/resource"
 	"github.com/hashicorp/terraform-plugin-testing/terraform"
 )
@@ -161,6 +165,31 @@ func environmentTemplate(t *testing.T, template string, otherVars ...map[string]
 	return commands.TrimLeadingWhitespace(template)
 }
 
+// ProvidersWithResourceFallbacks creates test providers, falling back to the SDKv2 provider for the
+// specified resources. This is a convenience constructor that ensures that the resulting mux'ed provider
+// uses the SDKv2 implementation for the specified resources.
+func ProvidersWithResourceFallbacks(resourceFallbacks []string) (*schema.Provider, provider.Provider) {
+	pluginfwOpt := pluginfw.WithSdkV2ResourceFallbacks(resourceFallbacks)
+	pluginFrameworkProvider := PluginFrameworkProviderForTest(pluginfwOpt)
+
+	sdkV2Opt := sdkv2.WithSdkV2ResourceFallbacks(resourceFallbacks)
+	sdkV2Provider := SdkV2ProviderForTest(sdkV2Opt)
+
+	return sdkV2Provider, pluginFrameworkProvider
+}
+
+// SdkV2ProviderForTest creates a test provider with the default config customizer.
+func SdkV2ProviderForTest(sdkV2Options ...sdkv2.SdkV2ProviderOption) *schema.Provider {
+	opts := append(sdkV2Options, sdkv2.WithConfigCustomizer(DefaultConfigCustomizer))
+	return sdkv2.DatabricksProvider(opts...)
+}
+
+// PluginFrameworkProviderForTest creates a test provider with the default config customizer.
+func PluginFrameworkProviderForTest(pluginFwOptions ...pluginfw.PluginFrameworkOption) provider.Provider {
+	opts := append(pluginFwOptions, pluginfw.WithConfigCustomizer(DefaultConfigCustomizer))
+	return pluginfw.GetDatabricksProviderPluginFramework(opts...)
+}
+
 // Test wrapper over terraform testing framework. Multiple steps share the same
 // terraform state context.
 func run(t *testing.T, steps []Step) {
@@ -173,7 +202,12 @@ func run(t *testing.T, steps []Step) {
 		"databricks": func() (tfprotov6.ProviderServer, error) {
 			ctx := context.Background()
 
-			return providers.GetProviderServer(ctx)
+			// The SDKv2 and Plugin Framework providers are customized to increase the default HTTP timeout.
+			// Otherwise, they are no different from the production provider.
+			sdkPluginProvider := SdkV2ProviderForTest()
+			pluginFrameworkProvider := PluginFrameworkProviderForTest()
+
+			return providers.GetProviderServer(ctx, providers.WithSdkV2Provider(sdkPluginProvider), providers.WithPluginFrameworkProvider(pluginFrameworkProvider))
 		},
 	}
 	cwd, err := os.Getwd()
@@ -252,6 +286,14 @@ func run(t *testing.T, steps []Step) {
 	})
 }
 
+// DefaultConfigCustomizer modifies the SDK configuration, setting the HTTP timeout to 10 minutes.
+// Most APIs have a default timeout of 1 minute, but some have longer. Extending the HTTP timeout
+// ensures that tests don't fail even if individual requests take longer than 1 minute.
+func DefaultConfigCustomizer(cfg *config.Config) error {
+	cfg.HTTPTimeoutSeconds = 10 * 60
+	return nil
+}
+
 // resourceCheck calls back a function with client and resource id
 func ResourceCheck(name string,
 	cb func(ctx context.Context, client *common.DatabricksClient, id string) error) resource.TestCheckFunc {

internal/providers/pluginfw/pluginfw.go

@@ -26,23 +26,33 @@ import (
 	"github.com/hashicorp/terraform-plugin-log/tflog"
 )
 
-func GetDatabricksProviderPluginFramework(sdkV2FallbackOptions ...SdkV2FallbackOption) provider.Provider {
-	p := &DatabricksProviderPluginFramework{sdkV2Fallbacks: sdkV2FallbackOptions}
+func GetDatabricksProviderPluginFramework(opts ...PluginFrameworkOption) provider.Provider {
+	providerOptions := &pluginFrameworkOptions{}
+	for _, opt := range opts {
+		opt.Apply(providerOptions)
+	}
+	p := &DatabricksProviderPluginFramework{
+		sdkV2ResourceFallbacks:   providerOptions.resourceFallbacks,
+		sdkV2DataSourceFallbacks: providerOptions.dataSourceFallbacks,
+		configCustomizer:         providerOptions.configCustomizer,
+	}
 	return p
 }
 
 type DatabricksProviderPluginFramework struct {
-	sdkV2Fallbacks []SdkV2FallbackOption
+	sdkV2ResourceFallbacks   []string
+	sdkV2DataSourceFallbacks []string
+	configCustomizer         func(*config.Config) error
 }
 
 var _ provider.Provider = (*DatabricksProviderPluginFramework)(nil)
 
 func (p *DatabricksProviderPluginFramework) Resources(ctx context.Context) []func() resource.Resource {
-	return getPluginFrameworkResourcesToRegister(p.sdkV2Fallbacks...)
+	return getPluginFrameworkResourcesToRegister(p.sdkV2ResourceFallbacks)
 }
 
 func (p *DatabricksProviderPluginFramework) DataSources(ctx context.Context) []func() datasource.DataSource {
-	return getPluginFrameworkDataSourcesToRegister(p.sdkV2Fallbacks...)
+	return getPluginFrameworkDataSourcesToRegister(p.sdkV2DataSourceFallbacks)
 }
 
 func (p *DatabricksProviderPluginFramework) Schema(ctx context.Context, req provider.SchemaRequest, resp *provider.SchemaResponse) {
@@ -55,7 +65,7 @@ func (p *DatabricksProviderPluginFramework) Metadata(ctx context.Context, req pr
 }
 
 func (p *DatabricksProviderPluginFramework) Configure(ctx context.Context, req provider.ConfigureRequest, resp *provider.ConfigureResponse) {
-	client := configureDatabricksClient_PluginFramework(ctx, req, resp)
+	client := p.configureDatabricksClient(ctx, req, resp)
 	resp.DataSourceData = client
 	resp.ResourceData = client
 }
@@ -88,7 +98,7 @@ func providerSchemaPluginFramework() schema.Schema {
 	}
 }
 
-func configureDatabricksClient_PluginFramework(ctx context.Context, req provider.ConfigureRequest, resp *provider.ConfigureResponse) any {
+func (p *DatabricksProviderPluginFramework) configureDatabricksClient(ctx context.Context, req provider.ConfigureRequest, resp *provider.ConfigureResponse) any {
 	cfg := &config.Config{}
 	attrsUsed := []string{}
 	for _, attr := range config.ConfigAttributes {
@@ -150,6 +160,13 @@ func configureDatabricksClient_PluginFramework(ctx context.Context, req provider
 			cfg.AuthType = newer
 		}
 	}
+	if p.configCustomizer != nil {
+		err := p.configCustomizer(cfg)
+		if err != nil {
+			resp.Diagnostics.AddError("Failed to customize config", err.Error())
+			return nil
+		}
+	}
 	client, err := client.New(cfg)
 	if err != nil {
 		resp.Diagnostics.Append(diag.NewErrorDiagnostic(err.Error(), ""))

internal/providers/pluginfw/pluginfw_rollout_utils.go

@@ -12,6 +12,7 @@ import (
 	"slices"
 	"strings"
 
+	"github.com/databricks/databricks-sdk-go/config"
 	"github.com/databricks/terraform-provider-databricks/internal/providers/pluginfw/products/app"
 	"github.com/databricks/terraform-provider-databricks/internal/providers/pluginfw/products/catalog"
 	"github.com/databricks/terraform-provider-databricks/internal/providers/pluginfw/products/cluster"
@@ -70,42 +71,56 @@ var pluginFwOnlyDataSources = append(
 	autoGeneratedDataSources...,
 )
 
-type sdkV2FallbackOptions struct {
+type pluginFrameworkOptions struct {
 	resourceFallbacks   []string
 	dataSourceFallbacks []string
+	configCustomizer    func(*config.Config) error
 }
 
-// SdkV2FallbackOption is an interface for acceptance tests to specify resources / data sources to fallback to SDK V2
-type SdkV2FallbackOption interface {
-	Apply(*sdkV2FallbackOptions)
+// PluginFrameworkOption is an interface for acceptance tests to specify resources / data sources to fallback to SDK V2
+type PluginFrameworkOption interface {
+	Apply(*pluginFrameworkOptions)
 }
 
 type sdkV2ResourceFallback struct {
 	resourceFallbacks []string
 }
 
-func (o *sdkV2ResourceFallback) Apply(options *sdkV2FallbackOptions) {
+func (o *sdkV2ResourceFallback) Apply(options *pluginFrameworkOptions) {
 	options.resourceFallbacks = o.resourceFallbacks
 }
 
 // WithSdkV2ResourceFallbacks is a helper function to specify resources to fallback to SDK V2
-func WithSdkV2ResourceFallbacks(fallbacks ...string) SdkV2FallbackOption {
+func WithSdkV2ResourceFallbacks(fallbacks []string) PluginFrameworkOption {
 	return &sdkV2ResourceFallback{resourceFallbacks: fallbacks}
 }
 
 type sdkv2DataSourceFallback struct {
 	dataSourceFallbacks []string
 }
 
-func (o *sdkv2DataSourceFallback) Apply(options *sdkV2FallbackOptions) {
+func (o *sdkv2DataSourceFallback) Apply(options *pluginFrameworkOptions) {
 	options.dataSourceFallbacks = o.dataSourceFallbacks
 }
 
 // WithSdkV2DataSourceFallbacks is a helper function to specify data sources to fallback to SDK V2
-func WithSdkV2DataSourceFallbacks(fallbacks []string) SdkV2FallbackOption {
+func WithSdkV2DataSourceFallbacks(fallbacks []string) PluginFrameworkOption {
 	return &sdkv2DataSourceFallback{dataSourceFallbacks: fallbacks}
 }
 
+type configCustomizer struct {
+	configCustomizer func(*config.Config) error
+}
+
+func (o *configCustomizer) Apply(options *pluginFrameworkOptions) {
+	options.configCustomizer = o.configCustomizer
+}
+
+// WithConfigCustomizer allows the caller to customize the SDK config after config resolution.
+func WithConfigCustomizer(customizer func(*config.Config) error) PluginFrameworkOption {
+	return &configCustomizer{configCustomizer: customizer}
+}
+
 // GetUseSdkV2DataSources is a helper function to get name of resources that should use SDK V2 instead of plugin framework
 func getUseSdkV2Resources() []string {
 	useSdkV2 := os.Getenv("USE_SDK_V2_RESOURCES")
@@ -137,18 +152,13 @@ func shouldUseSdkV2DataSource(dataSourceName string) bool {
 }
 
 // getPluginFrameworkResourcesToRegister is a helper function to get the list of resources that are migrated away from sdkv2 to plugin framework
-func getPluginFrameworkResourcesToRegister(sdkV2Fallbacks ...SdkV2FallbackOption) []func() resource.Resource {
-	fallbackOption := sdkV2FallbackOptions{}
-	for _, o := range sdkV2Fallbacks {
-		o.Apply(&fallbackOption)
-	}
-
+func getPluginFrameworkResourcesToRegister(resourceFallbacks []string) []func() resource.Resource {
 	var resources []func() resource.Resource
 
 	// Loop through the map and add resources if they're not specifically marked to use the SDK V2
 	for _, resourceFunc := range migratedResources {
 		name := getResourceName(resourceFunc)
-		if !shouldUseSdkV2Resource(name) && !slices.Contains(fallbackOption.resourceFallbacks, name) {
+		if !shouldUseSdkV2Resource(name) && !slices.Contains(resourceFallbacks, name) {
 			resources = append(resources, resourceFunc)
 		}
 	}
@@ -157,18 +167,13 @@ func getPluginFrameworkResourcesToRegister(sdkV2Fallbacks ...SdkV2FallbackOption
 }
 
 // getPluginFrameworkDataSourcesToRegister is a helper function to get the list of data sources that are migrated away from sdkv2 to plugin framework
-func getPluginFrameworkDataSourcesToRegister(sdkV2Fallbacks ...SdkV2FallbackOption) []func() datasource.DataSource {
-	fallbackOption := sdkV2FallbackOptions{}
-	for _, o := range sdkV2Fallbacks {
-		o.Apply(&fallbackOption)
-	}
-
+func getPluginFrameworkDataSourcesToRegister(dataSourceFallbacks []string) []func() datasource.DataSource {
 	var dataSources []func() datasource.DataSource
 
 	// Loop through the map and add data sources if they're not specifically marked to use the SDK V2
 	for _, dataSourceFunc := range migratedDataSources {
 		name := getDataSourceName(dataSourceFunc)
-		if !shouldUseSdkV2DataSource(name) && !slices.Contains(fallbackOption.dataSourceFallbacks, name) {
+		if !shouldUseSdkV2DataSource(name) && !slices.Contains(dataSourceFallbacks, name) {
 			dataSources = append(dataSources, dataSourceFunc)
 		}
 	}
@@ -189,33 +194,23 @@ func getDataSourceName(dataSourceFunc func() datasource.DataSource) string {
 }
 
 // GetSdkV2ResourcesToRemove is a helper function to get the list of resources that are migrated away from sdkv2 to plugin framework
-func GetSdkV2ResourcesToRemove(sdkV2Fallbacks ...SdkV2FallbackOption) []string {
-	fallbackOption := sdkV2FallbackOptions{}
-	for _, o := range sdkV2Fallbacks {
-		o.Apply(&fallbackOption)
-	}
-
+func GetSdkV2ResourcesToRemove(resourceFallbacks []string) []string {
 	resourcesToRemove := []string{}
 	for _, resourceFunc := range migratedResources {
 		name := getResourceName(resourceFunc)
-		if !shouldUseSdkV2Resource(name) && !slices.Contains(fallbackOption.resourceFallbacks, name) {
+		if !shouldUseSdkV2Resource(name) && !slices.Contains(resourceFallbacks, name) {
 			resourcesToRemove = append(resourcesToRemove, name)
 		}
 	}
 	return resourcesToRemove
 }
 
 // GetSdkV2DataSourcesToRemove is a helper function to get the list of data sources that are migrated away from sdkv2 to plugin framework
-func GetSdkV2DataSourcesToRemove(sdkV2Fallbacks ...SdkV2FallbackOption) []string {
-	fallbackOption := sdkV2FallbackOptions{}
-	for _, o := range sdkV2Fallbacks {
-		o.Apply(&fallbackOption)
-	}
-
+func GetSdkV2DataSourcesToRemove(dataSourceFallbacks []string) []string {
 	dataSourcesToRemove := []string{}
 	for _, dataSourceFunc := range migratedDataSources {
 		name := getDataSourceName(dataSourceFunc)
-		if !shouldUseSdkV2DataSource(name) && !slices.Contains(fallbackOption.dataSourceFallbacks, name) {
+		if !shouldUseSdkV2DataSource(name) && !slices.Contains(dataSourceFallbacks, name) {
 			dataSourcesToRemove = append(dataSourcesToRemove, name)
 		}
 	}

internal/providers/pluginfw/products/library/resource_library_acc_test.go

@@ -12,7 +12,6 @@ import (
 	"github.com/databricks/databricks-sdk-go/service/compute"
 	"github.com/databricks/terraform-provider-databricks/internal/acceptance"
 	"github.com/databricks/terraform-provider-databricks/internal/providers"
-	"github.com/databricks/terraform-provider-databricks/internal/providers/pluginfw"
 	"github.com/hashicorp/terraform-plugin-go/tfprotov6"
 	"github.com/hashicorp/terraform-plugin-testing/terraform"
 )
@@ -156,7 +155,8 @@ func TestAccLibraryUpdate(t *testing.T) {
 
 var sdkV2FallbackFactory = map[string]func() (tfprotov6.ProviderServer, error){
 	"databricks": func() (tfprotov6.ProviderServer, error) {
-		return providers.GetProviderServer(context.Background(), providers.WithSdkV2FallbackOptions(pluginfw.WithSdkV2ResourceFallbacks("databricks_library")))
+		sdkv2Provider, pluginfwProvider := acceptance.ProvidersWithResourceFallbacks([]string{"databricks_library"})
+		return providers.GetProviderServer(context.Background(), providers.WithSdkV2Provider(sdkv2Provider), providers.WithPluginFrameworkProvider(pluginfwProvider))
 	},
 }
 

internal/providers/pluginfw/products/qualitymonitor/resource_quality_monitor_acc_test.go

@@ -7,7 +7,6 @@ import (
 
 	"github.com/databricks/terraform-provider-databricks/internal/acceptance"
 	"github.com/databricks/terraform-provider-databricks/internal/providers"
-	"github.com/databricks/terraform-provider-databricks/internal/providers/pluginfw"
 	"github.com/hashicorp/terraform-plugin-go/tfprotov6"
 )
 
@@ -168,7 +167,8 @@ func TestUcAccUpdateQualityMonitor(t *testing.T) {
 
 var sdkV2FallbackFactory = map[string]func() (tfprotov6.ProviderServer, error){
 	"databricks": func() (tfprotov6.ProviderServer, error) {
-		return providers.GetProviderServer(context.Background(), providers.WithSdkV2FallbackOptions(pluginfw.WithSdkV2ResourceFallbacks("databricks_quality_monitor")))
+		sdkv2Provider, pluginfwProvider := acceptance.ProvidersWithResourceFallbacks([]string{"databricks_quality_monitor"})
+		return providers.GetProviderServer(context.Background(), providers.WithSdkV2Provider(sdkv2Provider), providers.WithPluginFrameworkProvider(pluginfwProvider))
 	},
 }
 

internal/providers/providers.go

@@ -20,7 +20,6 @@ import (
 type serverOptions struct {
 	sdkV2Provider           *schema.Provider
 	pluginFrameworkProvider provider.Provider
-	sdkV2fallbacks          []pluginfw.SdkV2FallbackOption
 }
 
 // ServerOption is a common interface for overriding providers in GetProviderServer functino call.
@@ -42,18 +41,18 @@ func WithSdkV2Provider(sdkV2Provider *schema.Provider) ServerOption {
 	return &sdkV2ProviderOption{sdkV2Provider: sdkV2Provider}
 }
 
-type sdkV2FallbackOption struct {
-	sdkV2fallbacks []pluginfw.SdkV2FallbackOption
+type pluginFrameworkProviderOption struct {
+	pluginFrameworkProvider provider.Provider
 }
 
-func (o *sdkV2FallbackOption) Apply(options *serverOptions) {
-	options.sdkV2fallbacks = o.sdkV2fallbacks
+func (o *pluginFrameworkProviderOption) Apply(options *serverOptions) {
+	options.pluginFrameworkProvider = o.pluginFrameworkProvider
 }
 
-// WithSdkV2FallbackOptions allows overriding the SDKv2 fallback options used when creating a Terraform provider with muxing.
-// This is typically used in acceptance test for testing the compatibility between sdkv2 and plugin framework.
-func WithSdkV2FallbackOptions(options ...pluginfw.SdkV2FallbackOption) ServerOption {
-	return &sdkV2FallbackOption{sdkV2fallbacks: options}
+// WithPluginFrameworkProvider allows overriding the plugin framework provider used when creating a Terraform provider with muxing.
+// This is typically used in acceptance test for a test step to have a custom provider override.
+func WithPluginFrameworkProvider(pluginFrameworkProvider provider.Provider) ServerOption {
+	return &pluginFrameworkProviderOption{pluginFrameworkProvider: pluginFrameworkProvider}
 }
 
 // GetProviderServer initializes and returns a Terraform Protocol v6 ProviderServer.
@@ -75,11 +74,11 @@ func GetProviderServer(ctx context.Context, options ...ServerOption) (tfprotov6.
 	}
 	sdkPluginProvider := serverOptions.sdkV2Provider
 	if sdkPluginProvider == nil {
-		sdkPluginProvider = sdkv2.DatabricksProvider(serverOptions.sdkV2fallbacks...)
+		sdkPluginProvider = sdkv2.DatabricksProvider()
 	}
 	pluginFrameworkProvider := serverOptions.pluginFrameworkProvider
 	if pluginFrameworkProvider == nil {
-		pluginFrameworkProvider = pluginfw.GetDatabricksProviderPluginFramework(serverOptions.sdkV2fallbacks...)
+		pluginFrameworkProvider = pluginfw.GetDatabricksProviderPluginFramework()
 	}
 
 	upgradedSdkPluginProvider, err := tf5to6server.UpgradeServer(