fix: Terraform state path for disabled workspaces

internal/exec/yaml_func_terraform_state_workspaces_disabled_test.go

@@ -0,0 +1,194 @@
+package exec
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	log "github.com/cloudposse/atmos/pkg/logger"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	cfg "github.com/cloudposse/atmos/pkg/config"
+	"github.com/cloudposse/atmos/pkg/schema"
+	u "github.com/cloudposse/atmos/pkg/utils"
+)
+
+// TestYamlFuncTerraformStateWorkspacesDisabled tests that the !terraform.state YAML function
+// works correctly when Terraform workspaces are disabled (workspaces_enabled: false).
+//
+// When workspaces are disabled:
+// - The workspace name is "default"
+// - For local backend: state is stored at terraform.tfstate (not terraform.tfstate.d/default/terraform.tfstate)
+// - For S3 backend: state is stored at <key> (not <workspace_key_prefix>/default/<key>)
+//
+// See: https://github.com/cloudposse/atmos/issues/1920
+func TestYamlFuncTerraformStateWorkspacesDisabled(t *testing.T) {
+	err := os.Unsetenv("ATMOS_CLI_CONFIG_PATH")
+	if err != nil {
+		t.Fatalf("Failed to unset 'ATMOS_CLI_CONFIG_PATH': %v", err)
+	}
+
+	err = os.Unsetenv("ATMOS_BASE_PATH")
+	if err != nil {
+		t.Fatalf("Failed to unset 'ATMOS_BASE_PATH': %v", err)
+	}
+
+	log.SetLevel(log.InfoLevel)
+	log.SetOutput(os.Stdout)
+
+	stack := "test"
+
+	defer func() {
+		// Delete the generated files and folders after the test.
+		mockComponentPath := filepath.Join("..", "..", "tests", "fixtures", "components", "terraform", "mock")
+		// Clean up terraform state files.
+		err := os.RemoveAll(filepath.Join(mockComponentPath, ".terraform"))
+		assert.NoError(t, err)
+
+		err = os.RemoveAll(filepath.Join(mockComponentPath, "terraform.tfstate.d"))
+		assert.NoError(t, err)
+
+		// When workspaces are disabled, state is stored at terraform.tfstate (not in terraform.tfstate.d/).
+		err = os.Remove(filepath.Join(mockComponentPath, "terraform.tfstate"))
+		// Ignore error if file doesn't exist.
+		if err != nil && !os.IsNotExist(err) {
+			assert.NoError(t, err)
+		}
+
+		err = os.Remove(filepath.Join(mockComponentPath, "terraform.tfstate.backup"))
+		// Ignore error if file doesn't exist.
+		if err != nil && !os.IsNotExist(err) {
+			assert.NoError(t, err)
+		}
+	}()
+
+	// Define the working directory (workspaces-disabled fixture).
+	workDir := "../../tests/fixtures/scenarios/atmos-terraform-state-yaml-function-workspaces-disabled"
+	t.Chdir(workDir)
+
+	// Deploy component-1 first to create terraform state.
+	info := schema.ConfigAndStacksInfo{
+		StackFromArg:     "",
+		Stack:            stack,
+		StackFile:        "",
+		ComponentType:    "terraform",
+		ComponentFromArg: "component-1",
+		SubCommand:       "deploy",
+		ProcessTemplates: true,
+		ProcessFunctions: true,
+	}
+
+	err = ExecuteTerraform(info)
+	require.NoError(t, err, "Failed to execute 'ExecuteTerraform' for component-1")
+
+	// Initialize CLI config.
+	atmosConfig, err := cfg.InitCliConfig(info, true)
+	require.NoError(t, err)
+
+	// Verify that workspaces are disabled.
+	require.NotNil(t, atmosConfig.Components.Terraform.WorkspacesEnabled)
+	assert.False(t, *atmosConfig.Components.Terraform.WorkspacesEnabled,
+		"Expected workspaces to be disabled in this test fixture")
+
+	// Test !terraform.state can read outputs from component-1.
+	// When workspaces are disabled, the state should be at terraform.tfstate (not terraform.tfstate.d/default/terraform.tfstate).
+	d, err := processTagTerraformState(&atmosConfig, "!terraform.state component-1 foo", stack, nil)
+	require.NoError(t, err)
+	assert.Equal(t, "component-1-a", d, "Expected to read 'foo' output from component-1 state")
+
+	d, err = processTagTerraformState(&atmosConfig, "!terraform.state component-1 bar", stack, nil)
+	require.NoError(t, err)
+	assert.Equal(t, "component-1-b", d, "Expected to read 'bar' output from component-1 state")
+
+	d, err = processTagTerraformState(&atmosConfig, "!terraform.state component-1 test baz", "", nil)
+	require.NoError(t, err)
+	assert.Equal(t, "component-1-c", d, "Expected to read 'baz' output from component-1 state")
+
+	// Verify component-2 can use !terraform.state to reference component-1's outputs.
+	res, err := ExecuteDescribeComponent(&ExecuteDescribeComponentParams{
+		Component:            "component-2",
+		Stack:                stack,
+		ProcessTemplates:     true,
+		ProcessYamlFunctions: true,
+		Skip:                 nil,
+		AuthManager:          nil,
+	})
+	require.NoError(t, err)
+
+	y, err := u.ConvertToYAML(res)
+	require.NoError(t, err)
+	assert.Contains(t, y, "foo: component-1-a", "component-2 should have foo from component-1 state")
+	assert.Contains(t, y, "bar: component-1-b", "component-2 should have bar from component-1 state")
+	assert.Contains(t, y, "baz: component-1-c", "component-2 should have baz from component-1 state")
+}
+
+// TestWorkspacesDisabledStateLocation verifies that when workspaces are disabled,
+// the terraform state is stored at the correct location (terraform.tfstate, not terraform.tfstate.d/default/terraform.tfstate).
+func TestWorkspacesDisabledStateLocation(t *testing.T) {
+	err := os.Unsetenv("ATMOS_CLI_CONFIG_PATH")
+	require.NoError(t, err)
+
+	err = os.Unsetenv("ATMOS_BASE_PATH")
+	require.NoError(t, err)
+
+	log.SetLevel(log.InfoLevel)
+	log.SetOutput(os.Stdout)
+
+	stack := "test"
+
+	// Get the absolute path to the mock component before changing directories.
+	// The path is relative to the current working directory (internal/exec).
+	mockComponentPath, err := filepath.Abs("../../tests/fixtures/components/terraform/mock")
+	require.NoError(t, err)
+
+	defer func() {
+		// Clean up terraform state files.
+		err := os.RemoveAll(filepath.Join(mockComponentPath, ".terraform"))
+		assert.NoError(t, err)
+
+		err = os.RemoveAll(filepath.Join(mockComponentPath, "terraform.tfstate.d"))
+		assert.NoError(t, err)
+
+		err = os.Remove(filepath.Join(mockComponentPath, "terraform.tfstate"))
+		if err != nil && !os.IsNotExist(err) {
+			assert.NoError(t, err)
+		}
+
+		err = os.Remove(filepath.Join(mockComponentPath, "terraform.tfstate.backup"))
+		if err != nil && !os.IsNotExist(err) {
+			assert.NoError(t, err)
+		}
+	}()
+
+	// Define the working directory (workspaces-disabled fixture).
+	workDir := "../../tests/fixtures/scenarios/atmos-terraform-state-yaml-function-workspaces-disabled"
+	t.Chdir(workDir)
+
+	// Deploy component-1.
+	info := schema.ConfigAndStacksInfo{
+		StackFromArg:     "",
+		Stack:            stack,
+		StackFile:        "",
+		ComponentType:    "terraform",
+		ComponentFromArg: "component-1",
+		SubCommand:       "deploy",
+		ProcessTemplates: true,
+		ProcessFunctions: true,
+	}
+
+	err = ExecuteTerraform(info)
+	require.NoError(t, err, "Failed to deploy component-1")
+
+	// Verify that the state file is at terraform.tfstate (not terraform.tfstate.d/default/terraform.tfstate).
+	stateFilePath := filepath.Join(mockComponentPath, "terraform.tfstate")
+	wrongStatePath := filepath.Join(mockComponentPath, "terraform.tfstate.d", "default", "terraform.tfstate")
+
+	// State should exist at the correct location.
+	_, err = os.Stat(stateFilePath)
+	assert.NoError(t, err, "State file should exist at %s when workspaces are disabled", stateFilePath)
+
+	// State should NOT exist at the wrong location.
+	_, err = os.Stat(wrongStatePath)
+	assert.True(t, os.IsNotExist(err), "State file should NOT exist at %s when workspaces are disabled", wrongStatePath)
+}

internal/terraform_backend/terraform_backend_local.go

@@ -13,21 +13,34 @@ import (
 
 // ReadTerraformBackendLocal reads the Terraform state file from the local backend.
 // If the state file does not exist, the function returns `nil`.
+//
+// According to Terraform local backend behavior:
+// - For the default workspace: state is stored at `terraform.tfstate`
+// - For named workspaces: state is stored at `terraform.tfstate.d/<workspace>/terraform.tfstate`
+//
+// See: https://github.com/cloudposse/atmos/issues/1920
 func ReadTerraformBackendLocal(
 	atmosConfig *schema.AtmosConfiguration,
 	componentSections *map[string]any,
 	_ *schema.AuthContext, // Auth context not used for local backend.
 ) ([]byte, error) {
 	defer perf.Track(atmosConfig, "terraform_backend.ReadTerraformBackendLocal")()
 
-	tfStateFilePath := filepath.Join(
+	workspace := GetTerraformWorkspace(componentSections)
+	componentPath := filepath.Join(
 		atmosConfig.TerraformDirAbsolutePath,
 		GetTerraformComponent(componentSections),
-		"terraform.tfstate.d",
-		GetTerraformWorkspace(componentSections),
-		"terraform.tfstate",
 	)
 
+	var tfStateFilePath string
+	if workspace == "" || workspace == "default" {
+		// Default workspace: state is stored directly at terraform.tfstate.
+		tfStateFilePath = filepath.Join(componentPath, "terraform.tfstate")
+	} else {
+		// Named workspace: state is stored at terraform.tfstate.d/<workspace>/terraform.tfstate.
+		tfStateFilePath = filepath.Join(componentPath, "terraform.tfstate.d", workspace, "terraform.tfstate")
+	}
+
 	// If the state file does not exist (the component in the stack has not been provisioned yet), return a `nil` result and no error.
 	if !u.FileExists(tfStateFilePath) {
 		return nil, nil

internal/terraform_backend/terraform_backend_local_test.go

@@ -98,3 +98,120 @@ func TestGetTerraformBackendLocal(t *testing.T) {
 		})
 	}
 }
+
+// TestReadTerraformBackendLocal_DefaultWorkspace verifies that when workspace
+// is "default" (meaning workspaces are disabled), the state file path should be
+// just terraform.tfstate, not terraform.tfstate.d/default/terraform.tfstate.
+//
+// This is based on Terraform local backend behavior:
+// - For the default workspace, state is stored directly at terraform.tfstate
+// - For named workspaces, state is stored at terraform.tfstate.d/<workspace>/terraform.tfstate
+//
+// See: https://github.com/cloudposse/atmos/issues/1920
+func TestReadTerraformBackendLocal_DefaultWorkspace(t *testing.T) {
+	tests := []struct {
+		name          string
+		workspace     string
+		stateLocation string // Where to place the state file (relative to component dir).
+		expected      string // Expected output value.
+	}{
+		{
+			name:          "default workspace - state at root",
+			workspace:     "default",
+			stateLocation: "terraform.tfstate",
+			expected:      "default-workspace-value",
+		},
+		{
+			name:          "empty workspace - state at root",
+			workspace:     "",
+			stateLocation: "terraform.tfstate",
+			expected:      "empty-workspace-value",
+		},
+		{
+			name:          "named workspace - state in workspace dir",
+			workspace:     "prod-us-east-1",
+			stateLocation: "terraform.tfstate.d/prod-us-east-1/terraform.tfstate",
+			expected:      "named-workspace-value",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tempDir := t.TempDir()
+			componentDir := filepath.Join(tempDir, "terraform", "test-component")
+
+			// Create the state file at the expected location.
+			stateFilePath := filepath.Join(componentDir, tt.stateLocation)
+			err := os.MkdirAll(filepath.Dir(stateFilePath), 0o755)
+			require.NoError(t, err)
+
+			stateContent := `{
+				"version": 4,
+				"terraform_version": "1.0.0",
+				"outputs": {
+					"test_output": {
+						"value": "` + tt.expected + `",
+						"type": "string"
+					}
+				}
+			}`
+			err = os.WriteFile(stateFilePath, []byte(stateContent), 0o644)
+			require.NoError(t, err)
+
+			config := &schema.AtmosConfiguration{
+				TerraformDirAbsolutePath: filepath.Join(tempDir, "terraform"),
+			}
+			componentData := map[string]any{
+				"component": "test-component",
+				"workspace": tt.workspace,
+			}
+
+			content, err := tb.ReadTerraformBackendLocal(config, &componentData, nil)
+			require.NoError(t, err)
+			require.NotNil(t, content, "Expected to find state file at %s", tt.stateLocation)
+
+			result, err := tb.ProcessTerraformStateFile(content)
+			require.NoError(t, err)
+			assert.Equal(t, tt.expected, result["test_output"],
+				"For workspace '%s', expected state from '%s'", tt.workspace, tt.stateLocation)
+		})
+	}
+}
+
+// TestReadTerraformBackendLocal_DefaultWorkspace_WrongLocation verifies that
+// when workspace is "default", we do NOT look in terraform.tfstate.d/default/.
+func TestReadTerraformBackendLocal_DefaultWorkspace_WrongLocation(t *testing.T) {
+	tempDir := t.TempDir()
+	componentDir := filepath.Join(tempDir, "terraform", "test-component")
+
+	// Create state file in the WRONG location (terraform.tfstate.d/default/).
+	wrongLocation := filepath.Join(componentDir, "terraform.tfstate.d", "default", "terraform.tfstate")
+	err := os.MkdirAll(filepath.Dir(wrongLocation), 0o755)
+	require.NoError(t, err)
+
+	stateContent := `{
+		"version": 4,
+		"terraform_version": "1.0.0",
+		"outputs": {
+			"test_output": {
+				"value": "wrong-location-value",
+				"type": "string"
+			}
+		}
+	}`
+	err = os.WriteFile(wrongLocation, []byte(stateContent), 0o644)
+	require.NoError(t, err)
+
+	config := &schema.AtmosConfiguration{
+		TerraformDirAbsolutePath: filepath.Join(tempDir, "terraform"),
+	}
+	componentData := map[string]any{
+		"component": "test-component",
+		"workspace": "default",
+	}
+
+	// Should NOT find the state file since it's in the wrong location.
+	content, err := tb.ReadTerraformBackendLocal(config, &componentData, nil)
+	require.NoError(t, err)
+	assert.Nil(t, content, "Should not find state file in terraform.tfstate.d/default/ for default workspace")
+}

internal/terraform_backend/terraform_backend_s3.go

@@ -120,13 +120,27 @@ func ReadTerraformBackendS3Internal(
 	defer perf.Track(nil, "terraform_backend.ReadTerraformBackendS3Internal")()
 
 	// Path to the tfstate file in the s3 bucket.
-	// S3 paths always use forward slashes, so path.Join is appropriate here.
-	//nolint:forbidigo // S3 paths require forward slashes regardless of OS
-	tfStateFilePath := path.Join(
-		GetBackendAttribute(backend, "workspace_key_prefix"),
-		GetTerraformWorkspace(componentSections),
-		GetBackendAttribute(backend, "key"),
-	)
+	// According to Terraform S3 backend documentation:
+	// - workspace_key_prefix is only used for non-default workspaces
+	// - For the default workspace, state is stored directly at the key path
+	// See: https://github.com/cloudposse/atmos/issues/1920
+	workspace := GetTerraformWorkspace(componentSections)
+	key := GetBackendAttribute(backend, "key")
+
+	var tfStateFilePath string
+	if workspace == "" || workspace == "default" {
+		// Default workspace: state is stored directly at the key path.
+		tfStateFilePath = key
+	} else {
+		// Named workspace: state is stored at workspace_key_prefix/workspace/key.
+		// S3 paths always use forward slashes, so path.Join is appropriate here.
+		//nolint:forbidigo // S3 paths require forward slashes regardless of OS
+		tfStateFilePath = path.Join(
+			GetBackendAttribute(backend, "workspace_key_prefix"),
+			workspace,
+			key,
+		)
+	}
 
 	bucket := GetBackendAttribute(backend, "bucket")