PSS: Implement initialisation of new working directory (or use of -reconfigure flag) while using state_store (#37732)

* Minor fixes in diagnostics

This can only be done once modules have been parsed and the required providers data is available. There are multiple places where config is parsed, into either Config or Module structs, so this needs to be implemented in multiple places.

* Rename test to make it specific to use of backend block in config

* Update initBackend to accept whole initArgs collection

* Only process --backend-config data, when setting up a `backend`, if that data isn't empty

* Simplify how mock provider factories are made in tests

* Update mock provider's default logic to track and manage existing workspaces

* Add `ProviderSchema` method to `Pluggable` structs. This allows calling code to access the provider schema when using provider configuration data.

* Add function for converting a providerreqs.Version to a hashicorp/go-version Version.

This is needed for using locks when creating the backend state file.

* Implement initial version of init new working directories using `stateStore_C_s`. Default to creating the default workspace if no workspaces exist.

* Update test fixtures to match the hashicorp/test mock provider used in PSS tests

* Allow tests to obtain locks that include `testingOverrides` providers.

The `testingOverrides` field will only be set in tests, so this should not impact end users.

* Add tests showing TF can initialize a working directory for the first time (and do the same when forced by -reconfigure flag). Remove replaced tests.

* Add -create-default-workspace flag, to be used to disable creating the default workspace by default when -input=false (i.e for use in CI). Refactor creation of default workspace logic. Add tests.

* Allow reattached providers to be used during init for PSS

* Rename variable to `backendHash` so relation to `backend` is clearer

* Allow `(m *Meta) Backend` to return warning diagnostics

* Protect against nil testingOverrides in providerFactoriesFromLocks

* Add test case seeing what happens if default workspace selected, doesn't exist, but other workspaces do exist.

The consequences here are due to using `selectWorkspace` in `stateStore_C_s`, matching what's done in `backend_C_r_s`.

* Address code consistency check failure on PR

* Refactor use of mock in test that's experiencing EOF error...

* Remove test that requires test to supply input for user prompt

This test passes when run in isolation but fails when run alongside other tests, even when skipping all other tests using `testStdinPipe`. I don't think the value of this test is great enough to start changing how we test stdin input.

* Allow -create-default-workspace to be used regardless of whether input is enabled or disabled

* Add TF_SKIP_CREATE_DEFAULT_WORKSPACE environment variable

* Responses to feedback, including making testStdinPipe helper log details of errors copying data to stdin.

Note: We cannot call t.Fatal from a non-test goroutine.

* Use Errorf instead

* Allow backend state files to not include version data when a builtin or reattached provider is in use.

* Add clarifying comment about re-attached providers when finding the matching entry in required_providers

* Report that the default workspace was created to the view

* Refactor: use error comparison via `errors.Is` to identify when no workspaces exist.

* Move handling of TF_ENABLE_PLUGGABLE_STATE_STORAGE into init's ParseInit func.

* Validate that PSS-related flags can only be used when experiments are enabled, enforce coupling of PSS-related flags when in use.

* Slight rewording of output message about default workspace

* Update test to assert new output about default workspace

Author: SarahFrench

internal/backend/pluggable/pluggable.go

@@ -68,6 +68,15 @@ func (p *Pluggable) ConfigSchema() *configschema.Block {
 	return val.Body
 }
 
+// ProviderSchema returns the schema for the provider implementing the state store.
+//
+// This isn't part of the backend.Backend interface but is needed in calling code.
+// When it's used the backend.Backend will need to be cast to a Pluggable.
+func (p *Pluggable) ProviderSchema() *configschema.Block {
+	schemaResp := p.provider.GetProviderSchema()
+	return schemaResp.Provider.Body
+}
+
 // PrepareConfig validates configuration for the state store in
 // the state storage provider. The configuration sent from Terraform core
 // will not include any values from environment variables; it is the

internal/backend/pluggable/pluggable_test.go

@@ -398,3 +398,70 @@ func TestPluggable_DeleteWorkspace(t *testing.T) {
 		t.Fatalf("expected error %q but got: %q", wantError, err)
 	}
 }
+
+func TestPluggable_ProviderSchema(t *testing.T) {
+	t.Run("Returns the expected provider schema", func(t *testing.T) {
+		mock := &testing_provider.MockProvider{
+			GetProviderSchemaResponse: &providers.GetProviderSchemaResponse{
+				Provider: providers.Schema{
+					Body: &configschema.Block{
+						Attributes: map[string]*configschema.Attribute{
+							"custom_attr": {Type: cty.String, Optional: true},
+						},
+					},
+				},
+			},
+		}
+		p, err := NewPluggable(mock, "foobar")
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+
+		// Calling code will need to case to Pluggable after using NewPluggable,
+		// so we do something similar in this test
+		var providerSchema *configschema.Block
+		if pluggable, ok := p.(*Pluggable); ok {
+			providerSchema = pluggable.ProviderSchema()
+		}
+
+		if !mock.GetProviderSchemaCalled {
+			t.Fatal("expected ProviderSchema to call the GetProviderSchema RPC")
+		}
+		if providerSchema == nil {
+			t.Fatal("ProviderSchema returned an unexpected nil schema")
+		}
+		if val := providerSchema.Attributes["custom_attr"]; val == nil {
+			t.Fatalf("expected the returned schema to include an attr called %q, but it was missing. Schema contains attrs: %v",
+				"custom_attr",
+				slices.Sorted(maps.Keys(providerSchema.Attributes)))
+		}
+	})
+
+	t.Run("Returns a nil schema when the provider has an empty schema", func(t *testing.T) {
+		mock := &testing_provider.MockProvider{
+			GetProviderSchemaResponse: &providers.GetProviderSchemaResponse{
+				Provider: providers.Schema{
+					// empty schema
+				},
+			},
+		}
+		p, err := NewPluggable(mock, "foobar")
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+
+		// Calling code will need to case to Pluggable after using NewPluggable,
+		// so we do something similar in this test
+		var providerSchema *configschema.Block
+		if pluggable, ok := p.(*Pluggable); ok {
+			providerSchema = pluggable.ProviderSchema()
+		}
+
+		if !mock.GetProviderSchemaCalled {
+			t.Fatal("expected ProviderSchema to call the GetProviderSchema RPC")
+		}
+		if providerSchema != nil {
+			t.Fatalf("expected ProviderSchema to return a nil schema but got: %#v", providerSchema)
+		}
+	})
+}

internal/command/arguments/init.go

@@ -4,6 +4,7 @@
 package arguments
 
 import (
+	"os"
 	"time"
 
 	"github.com/hashicorp/terraform/internal/tfdiags"
@@ -78,12 +79,16 @@ type Init struct {
 	// TODO(SarahFrench/radeksimko): Remove this once the feature is no longer
 	// experimental
 	EnablePssExperiment bool
+
+	// CreateDefaultWorkspace indicates whether the default workspace should be created by
+	// Terraform when initializing a state store for the first time.
+	CreateDefaultWorkspace bool
 }
 
 // ParseInit processes CLI arguments, returning an Init value and errors.
 // If errors are encountered, an Init value is still returned representing
 // the best effort interpretation of the arguments.
-func ParseInit(args []string) (*Init, tfdiags.Diagnostics) {
+func ParseInit(args []string, experimentsEnabled bool) (*Init, tfdiags.Diagnostics) {
 	var diags tfdiags.Diagnostics
 	init := &Init{
 		Vars: &Vars{},
@@ -111,6 +116,7 @@ func ParseInit(args []string) (*Init, tfdiags.Diagnostics) {
 	cmdFlags.BoolVar(&init.Json, "json", false, "json")
 	cmdFlags.Var(&init.BackendConfig, "backend-config", "")
 	cmdFlags.Var(&init.PluginPath, "plugin-dir", "plugin directory")
+	cmdFlags.BoolVar(&init.CreateDefaultWorkspace, "create-default-workspace", true, "when -input=false, use this flag to block creation of the default workspace")
 
 	// Used for enabling experimental code that's invoked before configuration is parsed.
 	cmdFlags.BoolVar(&init.EnablePssExperiment, "enable-pluggable-state-storage-experiment", false, "Enable the pluggable state storage experiment")
@@ -123,6 +129,46 @@ func ParseInit(args []string) (*Init, tfdiags.Diagnostics) {
 		))
 	}
 
+	if v := os.Getenv("TF_ENABLE_PLUGGABLE_STATE_STORAGE"); v != "" {
+		init.EnablePssExperiment = true
+	}
+
+	if v := os.Getenv("TF_SKIP_CREATE_DEFAULT_WORKSPACE"); v != "" {
+		// If TF_SKIP_CREATE_DEFAULT_WORKSPACE is set it will override
+		// a -create-default-workspace=true flag that's set explicitly,
+		// as that's indistinguishable from the default value being used.
+		init.CreateDefaultWorkspace = false
+	}
+
+	if !experimentsEnabled {
+		// If experiments aren't enabled then these flags should not be used.
+		if init.EnablePssExperiment {
+			diags = diags.Append(tfdiags.Sourceless(
+				tfdiags.Error,
+				"Cannot use -enable-pluggable-state-storage-experiment flag without experiments enabled",
+				"Terraform cannot use the-enable-pluggable-state-storage-experiment flag (or TF_ENABLE_PLUGGABLE_STATE_STORAGE environment variable) unless experiments are enabled.",
+			))
+		}
+		if !init.CreateDefaultWorkspace {
+			// Can only be set to false by using the flag
+			// and we cannot identify if -create-default-workspace=true is set explicitly.
+			diags = diags.Append(tfdiags.Sourceless(
+				tfdiags.Error,
+				"Cannot use -create-default-workspace flag without experiments enabled",
+				"Terraform cannot use the -create-default-workspace flag (or TF_SKIP_CREATE_DEFAULT_WORKSPACE environment variable) unless experiments are enabled.",
+			))
+		}
+	} else {
+		// Errors using flags despite experiments being enabled.
+		if !init.CreateDefaultWorkspace && !init.EnablePssExperiment {
+			diags = diags.Append(tfdiags.Sourceless(
+				tfdiags.Error,
+				"Cannot use -create-default-workspace=false flag unless the pluggable state storage experiment is enabled",
+				"Terraform cannot use the -create-default-workspace=false flag (or TF_SKIP_CREATE_DEFAULT_WORKSPACE environment variable) unless you also supply the -enable-pluggable-state-storage-experiment flag (or set the TF_ENABLE_PLUGGABLE_STATE_STORAGE environment variable).",
+			))
+		}
+	}
+
 	if init.MigrateState && init.Json {
 		diags = diags.Append(tfdiags.Sourceless(
 			tfdiags.Error,

internal/command/arguments/init_test.go

@@ -40,10 +40,11 @@ func TestParseInit_basicValid(t *testing.T) {
 					FlagName: "-backend-config",
 					Items:    &flagNameValue,
 				},
-				Vars:            &Vars{},
-				InputEnabled:    true,
-				CompactWarnings: false,
-				TargetFlags:     nil,
+				Vars:                   &Vars{},
+				InputEnabled:           true,
+				CompactWarnings:        false,
+				TargetFlags:            nil,
+				CreateDefaultWorkspace: true,
 			},
 		},
 		"setting multiple options": {
@@ -72,11 +73,12 @@ func TestParseInit_basicValid(t *testing.T) {
 					FlagName: "-backend-config",
 					Items:    &flagNameValue,
 				},
-				Vars:            &Vars{},
-				InputEnabled:    true,
-				Args:            []string{},
-				CompactWarnings: true,
-				TargetFlags:     nil,
+				Vars:                   &Vars{},
+				InputEnabled:           true,
+				Args:                   []string{},
+				CompactWarnings:        true,
+				TargetFlags:            nil,
+				CreateDefaultWorkspace: true,
 			},
 		},
 		"with cloud option": {
@@ -101,11 +103,12 @@ func TestParseInit_basicValid(t *testing.T) {
 					FlagName: "-backend-config",
 					Items:    &[]FlagNameValue{{Name: "-backend-config", Value: "backend.config"}},
 				},
-				Vars:            &Vars{},
-				InputEnabled:    false,
-				Args:            []string{},
-				CompactWarnings: false,
-				TargetFlags:     []string{"foo_bar.baz"},
+				Vars:                   &Vars{},
+				InputEnabled:           false,
+				Args:                   []string{},
+				CompactWarnings:        false,
+				TargetFlags:            []string{"foo_bar.baz"},
+				CreateDefaultWorkspace: true,
 			},
 		},
 	}
@@ -114,7 +117,8 @@ func TestParseInit_basicValid(t *testing.T) {
 
 	for name, tc := range testCases {
 		t.Run(name, func(t *testing.T) {
-			got, diags := ParseInit(tc.args)
+			experimentsEnabled := false
+			got, diags := ParseInit(tc.args, experimentsEnabled)
 			if len(diags) > 0 {
 				t.Fatalf("unexpected diags: %v", diags)
 			}
@@ -156,7 +160,8 @@ func TestParseInit_invalid(t *testing.T) {
 
 	for name, tc := range testCases {
 		t.Run(name, func(t *testing.T) {
-			got, diags := ParseInit(tc.args)
+			experimentsEnabled := false
+			got, diags := ParseInit(tc.args, experimentsEnabled)
 			if len(diags) == 0 {
 				t.Fatal("expected diags but got none")
 			}
@@ -170,6 +175,68 @@ func TestParseInit_invalid(t *testing.T) {
 	}
 }
 
+func TestParseInit_experimentalFlags(t *testing.T) {
+	testCases := map[string]struct {
+		args               []string
+		envs               map[string]string
+		wantErr            string
+		experimentsEnabled bool
+	}{
+		"error: -enable-pluggable-state-storage-experiment and experiments are disabled": {
+			args:               []string{"-enable-pluggable-state-storage-experiment"},
+			experimentsEnabled: false,
+			wantErr:            "Cannot use -enable-pluggable-state-storage-experiment flag without experiments enabled",
+		},
+		"error: TF_ENABLE_PLUGGABLE_STATE_STORAGE is set and experiments are disabled": {
+			envs: map[string]string{
+				"TF_ENABLE_PLUGGABLE_STATE_STORAGE": "1",
+			},
+			experimentsEnabled: false,
+			wantErr:            "Cannot use -enable-pluggable-state-storage-experiment flag without experiments enabled",
+		},
+		"error: -create-default-workspace=false and experiments are disabled": {
+			args:               []string{"-create-default-workspace=false"},
+			experimentsEnabled: false,
+			wantErr:            "Cannot use -create-default-workspace flag without experiments enabled",
+		},
+		"error: TF_SKIP_CREATE_DEFAULT_WORKSPACE is set and experiments are disabled": {
+			envs: map[string]string{
+				"TF_SKIP_CREATE_DEFAULT_WORKSPACE": "1",
+			},
+			experimentsEnabled: false,
+			wantErr:            "Cannot use -create-default-workspace flag without experiments enabled",
+		},
+		"error: -create-default-workspace=false used without -enable-pluggable-state-storage-experiment, while experiments are enabled": {
+			args:               []string{"-create-default-workspace=false"},
+			experimentsEnabled: true,
+			wantErr:            "Cannot use -create-default-workspace=false flag unless the pluggable state storage experiment is enabled",
+		},
+		"error: TF_SKIP_CREATE_DEFAULT_WORKSPACE used without -enable-pluggable-state-storage-experiment, while experiments are enabled": {
+			envs: map[string]string{
+				"TF_SKIP_CREATE_DEFAULT_WORKSPACE": "1",
+			},
+			experimentsEnabled: true,
+			wantErr:            "Cannot use -create-default-workspace=false flag unless the pluggable state storage experiment is enabled",
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			for k, v := range tc.envs {
+				t.Setenv(k, v)
+			}
+
+			_, diags := ParseInit(tc.args, tc.experimentsEnabled)
+			if len(diags) == 0 {
+				t.Fatal("expected diags but got none")
+			}
+			if got, want := diags.Err().Error(), tc.wantErr; !strings.Contains(got, want) {
+				t.Fatalf("wrong diags\n got: %s\nwant: %s", got, want)
+			}
+		})
+	}
+}
+
 func TestParseInit_vars(t *testing.T) {
 	testCases := map[string]struct {
 		args []string
@@ -207,7 +274,8 @@ func TestParseInit_vars(t *testing.T) {
 
 	for name, tc := range testCases {
 		t.Run(name, func(t *testing.T) {
-			got, diags := ParseInit(tc.args)
+			experimentsEnabled := false
+			got, diags := ParseInit(tc.args, experimentsEnabled)
 			if len(diags) > 0 {
 				t.Fatalf("unexpected diags: %v", diags)
 			}

internal/command/command_test.go

@@ -657,7 +657,10 @@ func testStdinPipe(t *testing.T, src io.Reader) func() {
 	// Copy the data from the reader to the pipe
 	go func() {
 		defer w.Close()
-		io.Copy(w, src)
+		_, err := io.Copy(w, src)
+		if err != nil {
+			t.Errorf("error when copying data from testStdinPipe reader argument to stdin: %s", err)
+		}
 	}()
 
 	return func() {