PSS: Add reusable method for obtaining the provider factory needed for accessing a state store. (#37665)

* Add method to allow accessing factories from locks that are in memory

* Create new getStateStoreProviderFactory method for accessing a factory from config

* Update getStateStoreProviderFactory to use in memory locks instead of reading them from the deps lock file. Update calling code to accommodate this.

* Add tests for getStateStoreProviderFactory, improve errors returned from method

* Update test following schema change in simple providers

* Move test case into e2etest package, so we protect against environments where building binaries isn't possible

* Fix issues with running test in e2etest package

* Update code comments

Co-authored-by: Radek Simko <[email protected]>

---------

Co-authored-by: Radek Simko <[email protected]>

Author: SarahFrench

internal/command/e2etest/meta_backend_test.go

@@ -0,0 +1,85 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package e2etest
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/apparentlymart/go-versions/versions"
+	tfaddr "github.com/hashicorp/terraform-registry-address"
+	"github.com/hashicorp/terraform/internal/addrs"
+	"github.com/hashicorp/terraform/internal/command"
+	"github.com/hashicorp/terraform/internal/configs"
+	"github.com/hashicorp/terraform/internal/depsfile"
+	"github.com/hashicorp/terraform/internal/e2e"
+	"github.com/hashicorp/terraform/internal/getproviders"
+	"github.com/hashicorp/terraform/internal/getproviders/providerreqs"
+)
+
+func TestMetaBackend_GetStateStoreProviderFactory(t *testing.T) {
+	t.Run("gets the matching factory from local provider cache", func(t *testing.T) {
+		if !canRunGoBuild {
+			// We're running in a separate-build-then-run context, so we can't
+			// currently execute this test which depends on being able to build
+			// new executable at runtime.
+			//
+			// (See the comment on canRunGoBuild's declaration for more information.)
+			t.Skip("can't run without building a new provider executable")
+		}
+
+		// Set up locks
+		locks := depsfile.NewLocks()
+		providerAddr := addrs.MustParseProviderSourceString("registry.terraform.io/hashicorp/simple")
+		constraint, err := providerreqs.ParseVersionConstraints(">1.0.0")
+		if err != nil {
+			t.Fatalf("test setup failed when making constraint: %s", err)
+		}
+		locks.SetProvider(
+			providerAddr,
+			versions.MustParseVersion("9.9.9"),
+			constraint,
+			[]providerreqs.Hash{""},
+		)
+
+		// Set up a local provider cache for the test to use
+		// 1. Build a binary for the current platform
+		simple6Provider := filepath.Join(".", "terraform-provider-simple6")
+		simple6ProviderExe := e2e.GoBuild("github.com/hashicorp/terraform/internal/provider-simple-v6/main", simple6Provider)
+		// 2. Create a working directory with .terraform/providers directory
+		td := t.TempDir()
+		t.Chdir(td)
+		providerPath := fmt.Sprintf(".terraform/providers/registry.terraform.io/hashicorp/simple/9.9.9/%s", getproviders.CurrentPlatform.String())
+		err = os.MkdirAll(providerPath, os.ModePerm)
+		if err != nil {
+			t.Fatal(err)
+		}
+		// 3. Move the binary into the cache folder created above.
+		os.Rename(simple6ProviderExe, fmt.Sprintf("%s/%s/terraform-provider-simple", td, providerPath))
+
+		config := &configs.StateStore{
+			ProviderAddr: tfaddr.MustParseProviderSource("registry.terraform.io/hashicorp/simple"),
+			// No other fields necessary for test.
+		}
+
+		// Setup the meta and test GetStateStoreProviderFactory
+		m := command.Meta{}
+		factory, diags := m.GetStateStoreProviderFactory(config, locks)
+		if diags.HasErrors() {
+			t.Fatalf("unexpected error : %s", err)
+		}
+
+		p, _ := factory()
+		defer p.Close()
+		s := p.GetProviderSchema()
+		expectedProviderDescription := "This is terraform-provider-simple v6"
+		if s.Provider.Body.Description != expectedProviderDescription {
+			t.Fatalf("expected description to be %q, but got %q", expectedProviderDescription, s.Provider.Body.Description)
+		}
+	})
+
+	// See command/meta_backend_test.go for other test cases
+}

internal/command/e2etest/providers_schema_test.go

@@ -71,6 +71,7 @@ func TestProvidersSchema(t *testing.T) {
             "provider": {
                 "version": 0,
                 "block": {
+                    "description": "This is terraform-provider-simple v5",
                     "description_kind": "plain"
                 }
             },
@@ -165,6 +166,7 @@ func TestProvidersSchema(t *testing.T) {
             "provider": {
                 "version": 0,
                 "block": {
+                    "description": "This is terraform-provider-simple v6",
                     "description_kind": "plain"
                 }
             },

internal/command/init.go

@@ -159,7 +159,7 @@ func (c *InitCommand) initCloud(ctx context.Context, root *configs.Module, extra
 	return back, true, diags
 }
 
-func (c *InitCommand) initBackend(ctx context.Context, root *configs.Module, extraConfig arguments.FlagNameValueSlice, viewType arguments.ViewType, view views.Init) (be backend.Backend, output bool, diags tfdiags.Diagnostics) {
+func (c *InitCommand) initBackend(ctx context.Context, root *configs.Module, extraConfig arguments.FlagNameValueSlice, viewType arguments.ViewType, configLocks *depsfile.Locks, view views.Init) (be backend.Backend, output bool, diags tfdiags.Diagnostics) {
 	ctx, span := tracer.Start(ctx, "initialize backend")
 	_ = ctx // prevent staticcheck from complaining to avoid a maintenance hazard of having the wrong ctx in scope here
 	defer span.End()
@@ -187,34 +187,9 @@ func (c *InitCommand) initBackend(ctx context.Context, root *configs.Module, ext
 		return nil, true, diags
 	case root.StateStore != nil:
 		// state_store config present
-		// Access provider factories
-		ctxOpts, err := c.contextOpts()
-		if err != nil {
-			diags = diags.Append(err)
-			return nil, true, diags
-		}
-
-		if root.StateStore.ProviderAddr.IsZero() {
-			// This should not happen; this data is populated when parsing config,
-			// even for builtin providers
-			panic(fmt.Sprintf("unknown provider while beginning to initialize state store %q from provider %q",
-				root.StateStore.Type,
-				root.StateStore.Provider.Name))
-		}
-
-		var exists bool
-		factory, exists := ctxOpts.Providers[root.StateStore.ProviderAddr]
-		if !exists {
-			diags = diags.Append(&hcl.Diagnostic{
-				Severity: hcl.DiagError,
-				Summary:  "Provider unavailable",
-				Detail: fmt.Sprintf("The provider %s (%q) is required to initialize the %q state store, but the matching provider factory is missing. This is a bug in Terraform and should be reported.",
-					root.StateStore.Provider.Name,
-					root.StateStore.ProviderAddr,
-					root.StateStore.Type,
-				),
-				Subject: &root.StateStore.TypeRange,
-			})
+		factory, fDiags := c.Meta.GetStateStoreProviderFactory(root.StateStore, configLocks)
+		diags = diags.Append(fDiags)
+		if fDiags.HasErrors() {
 			return nil, true, diags
 		}
 

internal/command/init_run.go

@@ -14,6 +14,7 @@ import (
 	"github.com/hashicorp/terraform/internal/command/arguments"
 	"github.com/hashicorp/terraform/internal/command/views"
 	"github.com/hashicorp/terraform/internal/configs"
+	"github.com/hashicorp/terraform/internal/depsfile"
 	"github.com/hashicorp/terraform/internal/states"
 	"github.com/hashicorp/terraform/internal/terraform"
 	"github.com/hashicorp/terraform/internal/tfdiags"
@@ -170,7 +171,10 @@ func (c *InitCommand) run(initArgs *arguments.Init, view views.Init) int {
 	case initArgs.Cloud && rootModEarly.CloudConfig != nil:
 		back, backendOutput, backDiags = c.initCloud(ctx, rootModEarly, initArgs.BackendConfig, initArgs.ViewType, view)
 	case initArgs.Backend:
-		back, backendOutput, backDiags = c.initBackend(ctx, rootModEarly, initArgs.BackendConfig, initArgs.ViewType, view)
+		// initBackend has new parameters that aren't relevant to the original (unpluggable) version of the init command logic here.
+		// So for this version of the init command, we pass in empty locks intentionally.
+		emptyLocks := depsfile.NewLocks()
+		back, backendOutput, backDiags = c.initBackend(ctx, rootModEarly, initArgs.BackendConfig, initArgs.ViewType, emptyLocks, view)
 	default:
 		// load the previously-stored backend config
 		back, backDiags = c.Meta.backendFromState(ctx)

internal/command/init_run_experiment.go

@@ -205,9 +205,7 @@ func (c *InitCommand) runPssInit(initArgs *arguments.Init, view views.Init) int
 	case initArgs.Cloud && rootModEarly.CloudConfig != nil:
 		back, backendOutput, backDiags = c.initCloud(ctx, rootModEarly, initArgs.BackendConfig, initArgs.ViewType, view)
 	case initArgs.Backend:
-		// TODO(SarahFrench/radeksimko) - pass information about config locks (`configLocks`) into initBackend to
-		// enable PSS
-		back, backendOutput, backDiags = c.initBackend(ctx, rootModEarly, initArgs.BackendConfig, initArgs.ViewType, view)
+		back, backendOutput, backDiags = c.initBackend(ctx, rootModEarly, initArgs.BackendConfig, initArgs.ViewType, configLocks, view)
 	default:
 		// load the previously-stored backend config
 		back, backDiags = c.Meta.backendFromState(ctx)

internal/command/meta.go

@@ -546,7 +546,7 @@ func (m *Meta) contextOpts() (*terraform.ContextOpts, error) {
 		opts.Provisioners = m.testingOverrides.Provisioners
 	} else {
 		var providerFactories map[addrs.Provider]providers.Factory
-		providerFactories, err = m.providerFactories()
+		providerFactories, err = m.ProviderFactories()
 		opts.Providers = providerFactories
 		opts.Provisioners = m.provisionerFactories()
 	}

internal/command/meta_backend.go

@@ -33,6 +33,7 @@ import (
 	"github.com/hashicorp/terraform/internal/command/views"
 	"github.com/hashicorp/terraform/internal/command/workdir"
 	"github.com/hashicorp/terraform/internal/configs"
+	"github.com/hashicorp/terraform/internal/depsfile"
 	"github.com/hashicorp/terraform/internal/didyoumean"
 	"github.com/hashicorp/terraform/internal/plans"
 	"github.com/hashicorp/terraform/internal/providers"
@@ -1746,6 +1747,57 @@ func (m *Meta) assertSupportedCloudInitOptions(mode cloud.ConfigChangeMode) tfdi
 	return diags
 }
 
+func (m *Meta) GetStateStoreProviderFactory(config *configs.StateStore, locks *depsfile.Locks) (providers.Factory, tfdiags.Diagnostics) {
+	var diags tfdiags.Diagnostics
+
+	if config == nil || locks == nil {
+		panic(fmt.Sprintf("nil config or nil locks passed to GetStateStoreProviderFactory: config %#v, locks %#v", config, locks))
+	}
+
+	if config.ProviderAddr.IsZero() {
+		// This should not happen; this data is populated when parsing config,
+		// even for builtin providers
+		return nil, diags.Append(&hcl.Diagnostic{
+			Severity: hcl.DiagError,
+			Summary:  "Unknown provider used for state storage",
+			Detail:   "Terraform could not find the provider used with the state_store. This is a bug in Terraform and should be reported.",
+			Subject:  &config.TypeRange,
+		})
+	}
+
+	factories, err := m.ProviderFactoriesFromLocks(locks)
+	if err != nil {
+		// This may happen if the provider isn't present in the provider cache.
+		// This should be caught earlier by logic that diffs the config against the backend state file.
+		return nil, diags.Append(&hcl.Diagnostic{
+			Severity: hcl.DiagError,
+			Summary:  "Provider unavailable",
+			Detail: fmt.Sprintf("Terraform experienced an error when trying to use provider %s (%q) to initialize the %q state store: %s",
+				config.Provider.Name,
+				config.ProviderAddr,
+				config.Type,
+				err),
+			Subject: &config.TypeRange,
+		})
+	}
+
+	factory, exists := factories[config.ProviderAddr]
+	if !exists {
+		return nil, diags.Append(&hcl.Diagnostic{
+			Severity: hcl.DiagError,
+			Summary:  "Provider unavailable",
+			Detail: fmt.Sprintf("The provider %s (%q) is required to initialize the %q state store, but the matching provider factory is missing. This is a bug in Terraform and should be reported.",
+				config.Provider.Name,
+				config.ProviderAddr,
+				config.Type,
+			),
+			Subject: &config.TypeRange,
+		})
+	}
+
+	return factory, diags
+}
+
 //-------------------------------------------------------------------
 // Output constants and initialization code
 //-------------------------------------------------------------------

internal/command/meta_backend_test.go

@@ -13,14 +13,18 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/apparentlymart/go-versions/versions"
 	"github.com/hashicorp/cli"
+	tfaddr "github.com/hashicorp/terraform-registry-address"
 	"github.com/hashicorp/terraform/internal/addrs"
 	"github.com/hashicorp/terraform/internal/backend"
 	"github.com/hashicorp/terraform/internal/cloud"
 	"github.com/hashicorp/terraform/internal/command/workdir"
 	"github.com/hashicorp/terraform/internal/configs"
 	"github.com/hashicorp/terraform/internal/configs/configschema"
 	"github.com/hashicorp/terraform/internal/copy"
+	"github.com/hashicorp/terraform/internal/depsfile"
+	"github.com/hashicorp/terraform/internal/getproviders/providerreqs"
 	"github.com/hashicorp/terraform/internal/plans"
 	"github.com/hashicorp/terraform/internal/providers"
 	testing_provider "github.com/hashicorp/terraform/internal/providers/testing"
@@ -2397,6 +2401,79 @@ func TestMetaBackend_configureStateStoreVariableUse(t *testing.T) {
 	}
 }
 
+func TestMetaBackend_GetStateStoreProviderFactory(t *testing.T) {
+	// See internal/command/e2etest/meta_backend_test.go for test case
+	// where a provider factory is found using a local provider cache
+
+	t.Run("returns an error if a matching factory can't be found", func(t *testing.T) {
+		// Set up locks
+		locks := depsfile.NewLocks()
+		providerAddr := addrs.MustParseProviderSourceString("registry.terraform.io/hashicorp/simple")
+		constraint, err := providerreqs.ParseVersionConstraints(">1.0.0")
+		if err != nil {
+			t.Fatalf("test setup failed when making constraint: %s", err)
+		}
+		locks.SetProvider(
+			providerAddr,
+			versions.MustParseVersion("9.9.9"),
+			constraint,
+			[]providerreqs.Hash{""},
+		)
+
+		config := &configs.StateStore{
+			ProviderAddr: tfaddr.MustParseProviderSource("registry.terraform.io/hashicorp/simple"),
+			Provider: &configs.Provider{
+				Name: "foobar",
+			},
+			Type: "store",
+		}
+
+		// Setup the meta and test providerFactoriesDuringInit
+		m := testMetaBackend(t, nil)
+		_, diags := m.GetStateStoreProviderFactory(config, locks)
+		if !diags.HasErrors() {
+			t.Fatalf("expected error but got none")
+		}
+		expectedErr := "Provider unavailable"
+		expectedDetail := "Terraform experienced an error when trying to use provider foobar (\"registry.terraform.io/hashicorp/simple\") to initialize the \"store\" state store"
+		if diags[0].Description().Summary != expectedErr {
+			t.Fatalf("expected error summary to include %q but got: %s",
+				expectedErr,
+				diags[0].Description().Summary,
+			)
+		}
+		if !strings.Contains(diags[0].Description().Detail, expectedDetail) {
+			t.Fatalf("expected error detail to include %q but got: %s",
+				expectedErr,
+				diags[0].Description().Detail,
+			)
+		}
+	})
+
+	t.Run("returns an error if provider addr data is missing", func(t *testing.T) {
+		// Only minimal locks needed
+		locks := depsfile.NewLocks()
+
+		config := &configs.StateStore{
+			ProviderAddr: tfaddr.Provider{}, // Empty
+		}
+
+		// Setup the meta and test providerFactoriesDuringInit
+		m := testMetaBackend(t, nil)
+		_, diags := m.GetStateStoreProviderFactory(config, locks)
+		if !diags.HasErrors() {
+			t.Fatal("expected and error but got none")
+		}
+		expectedErr := "Unknown provider used for state storage"
+		if !strings.Contains(diags.Err().Error(), expectedErr) {
+			t.Fatalf("expected error to include %q but got: %s",
+				expectedErr,
+				diags.Err().Error(),
+			)
+		}
+	})
+}
+
 func testMetaBackend(t *testing.T, args []string) *Meta {
 	var m Meta
 	m.Ui = new(cli.MockUi)