Implement schema mappings for modules with dashed fields (#427)

* Implement schema mappings for modules with dashed fields

* remove terraform artifacts used for debugging

Author: Zaid-Ajaj

pkg/modprovider/module.go

@@ -172,13 +172,60 @@ func (h *moduleHandler) prepSandbox(
 	// which will get further reused for Pulumi URNs.
 	tfName := getModuleName(urn)
 
+	hasOutputFieldMapping := inferredModule != nil &&
+		inferredModule.SchemaFieldMappings != nil &&
+		inferredModule.SchemaFieldMappings.OutputFieldMappings != nil
+
 	outputSpecs := []tfsandbox.TFOutputSpec{}
 	for outputName := range inferredModule.Outputs {
+		if hasOutputFieldMapping {
+			mappings := inferredModule.SchemaFieldMappings.OutputFieldMappings
+			if tfName, ok := mappings[outputName]; ok {
+				outputName = tfName
+			}
+		}
+
 		outputSpecs = append(outputSpecs, tfsandbox.TFOutputSpec{
 			Name: tfsandbox.DecodePulumiTopLevelKey(outputName),
 		})
 	}
 
+	// remap input fields to terraform module inputs
+	// for example if terraform module input was "input-value" but pulumi input was "input_value",
+	// then we need to remap it to "input-value" in the tf file.
+	hasInputFieldMappings := inferredModule != nil &&
+		inferredModule.SchemaFieldMappings != nil &&
+		inferredModule.SchemaFieldMappings.InputFieldMappings != nil
+
+	if hasInputFieldMappings {
+		mappings := inferredModule.SchemaFieldMappings.InputFieldMappings
+		for pulumiInputName, input := range moduleInputs {
+			if tfName, ok := mappings[pulumiInputName]; ok {
+				// if the input is mapped, use the mapped name
+				moduleInputs[tfName] = input
+				delete(moduleInputs, pulumiInputName)
+			}
+		}
+	}
+
+	// remap some required providers in the TF module. For example,
+	// if the module requires "google-beta", the Pulumi name of the field would be "google_beta"
+	// so we need to remap it to "google-beta" in the tf file.
+	hasProviderFieldMappings := inferredModule != nil &&
+		inferredModule.SchemaFieldMappings != nil &&
+		inferredModule.SchemaFieldMappings.ProviderFieldMappings != nil
+
+	if hasProviderFieldMappings {
+		mappings := inferredModule.SchemaFieldMappings.ProviderFieldMappings
+		for providerName, config := range providersConfig {
+			if tfName, ok := mappings[providerName]; ok {
+				// if the provider is mapped, use the mapped name
+				providersConfig[tfName] = config
+				delete(providersConfig, providerName)
+			}
+		}
+	}
+
 	err = tfsandbox.CreateTFFile(tfName, moduleSource,
 		moduleVersion, tf.WorkingDir(),
 		moduleInputs, outputSpecs, providersConfig)
@@ -202,7 +249,7 @@ func (h *moduleHandler) prepSandbox(
 	return tf, nil
 }
 
-// This method handles Create and Update in a uniform way; both map to tofu apply operation.
+// This method handles Create and Update in a uniform way; both map to tofu/terraform apply operation.
 func (h *moduleHandler) applyModuleOperation(
 	ctx context.Context,
 	urn urn.URN,
@@ -289,6 +336,22 @@ func (h *moduleHandler) applyModuleOperation(
 		logger.Log(ctx, tfsandbox.Error, fmt.Sprintf("partial failure in apply: %v", applyErr))
 	}
 
+	hasOutputFieldMappings := inferredModule != nil &&
+		inferredModule.SchemaFieldMappings != nil &&
+		inferredModule.SchemaFieldMappings.OutputFieldMappings != nil
+
+	if hasOutputFieldMappings {
+		mappings := inferredModule.SchemaFieldMappings.OutputFieldMappings
+		for tfName, output := range moduleOutputs {
+			for pulumiOutputName, mappedTerraformName := range mappings {
+				if tfName == mappedTerraformName {
+					moduleOutputs[pulumiOutputName] = output
+					delete(moduleOutputs, tfName)
+				}
+			}
+		}
+	}
+
 	return moduleOutputs, views, applyErr
 }
 

pkg/modprovider/tfmodules.go

@@ -92,13 +92,20 @@ func parseModuleSchemaOverrides(packageName string) []*ModuleSchemaOverride {
 	return overrides
 }
 
+type SchemaFieldMappings struct {
+	ProviderFieldMappings map[string]string
+	InputFieldMappings    map[resource.PropertyKey]resource.PropertyKey
+	OutputFieldMappings   map[resource.PropertyKey]resource.PropertyKey
+}
+
 type InferredModuleSchema struct {
-	Inputs          map[resource.PropertyKey]*schema.PropertySpec `json:"inputs"`
-	Outputs         map[resource.PropertyKey]*schema.PropertySpec `json:"outputs"`
-	SupportingTypes map[string]*schema.ComplexTypeSpec            `json:"supportingTypes"`
-	RequiredInputs  []resource.PropertyKey                        `json:"requiredInputs"`
-	NonNilOutputs   []resource.PropertyKey                        `json:"nonNilOutputs"`
-	ProvidersConfig schema.ConfigSpec                             `json:"providersConfig"`
+	Inputs              map[resource.PropertyKey]*schema.PropertySpec `json:"inputs"`
+	Outputs             map[resource.PropertyKey]*schema.PropertySpec `json:"outputs"`
+	SupportingTypes     map[string]*schema.ComplexTypeSpec            `json:"supportingTypes"`
+	RequiredInputs      []resource.PropertyKey                        `json:"requiredInputs"`
+	NonNilOutputs       []resource.PropertyKey                        `json:"nonNilOutputs"`
+	ProvidersConfig     schema.ConfigSpec                             `json:"providersConfig"`
+	SchemaFieldMappings *SchemaFieldMappings                          `json:"schemaFieldMappings,omitempty"`
 }
 
 var stringType = schema.TypeSpec{Type: "string"}
@@ -343,6 +350,10 @@ func InferModuleSchema(
 	return inferModuleSchema(ctx, tf, packageName, mod, ver, newComponentLogger(nil, nil))
 }
 
+func containsDash(s string) bool {
+	return strings.Contains(s, "-")
+}
+
 func inferModuleSchema(
 	ctx context.Context,
 	tf *tfsandbox.ModuleRuntime,
@@ -365,10 +376,26 @@ func inferModuleSchema(
 		ProvidersConfig: schema.ConfigSpec{
 			Variables: map[string]schema.PropertySpec{},
 		},
+		SchemaFieldMappings: &SchemaFieldMappings{
+			InputFieldMappings:    make(map[resource.PropertyKey]resource.PropertyKey),
+			OutputFieldMappings:   make(map[resource.PropertyKey]resource.PropertyKey),
+			ProviderFieldMappings: make(map[string]string),
+		},
 	}
 
+	providerFieldMappings := inferredModuleSchema.SchemaFieldMappings.ProviderFieldMappings
+	inputFieldMappings := inferredModuleSchema.SchemaFieldMappings.InputFieldMappings
+	outputFieldMappings := inferredModuleSchema.SchemaFieldMappings.OutputFieldMappings
+
 	if module.ProviderRequirements != nil {
 		for providerName := range module.ProviderRequirements.RequiredProviders {
+			if containsDash(providerName) {
+				// fields with dashes are not valid in Pulumi
+				// so we replace dashes with underscores
+				pulumiName := strings.ReplaceAll(providerName, "-", "_")
+				providerFieldMappings[pulumiName] = providerName
+				providerName = pulumiName
+			}
 			inferredModuleSchema.ProvidersConfig.Variables[providerName] = schema.PropertySpec{
 				Description: "provider configuration for " + providerName,
 				TypeSpec:    mapType(anyType),
@@ -377,6 +404,14 @@ func inferModuleSchema(
 	}
 
 	for variableName, variable := range module.Variables {
+		if containsDash(variableName) {
+			// fields with dashes are not valid in Pulumi
+			// so we replace dashes with underscores
+			pulumiName := strings.ReplaceAll(variableName, "-", "_")
+			inputFieldMappings[resource.PropertyKey(pulumiName)] = resource.PropertyKey(variableName)
+			variableName = pulumiName
+		}
+
 		variableType := convertType(variable.Type, variableName, packageName, inferredModuleSchema.SupportingTypes)
 
 		key := tfsandbox.PulumiTopLevelKey(variableName)
@@ -395,11 +430,21 @@ func inferModuleSchema(
 	}
 
 	for outputName, output := range module.Outputs {
+		if containsDash(outputName) {
+			// fields with dashes are not valid in Pulumi
+			// so we replace dashes with underscores
+			pulumiName := strings.ReplaceAll(outputName, "-", "_")
+			outputFieldMappings[resource.PropertyKey(pulumiName)] = resource.PropertyKey(outputName)
+			outputName = pulumiName
+		}
+
 		// TODO[pulumi/pulumi-terraform-module#70] reconsider output type inference vs config
 		var inferredType schema.TypeSpec
 		if referencedVariableName, ok := isVariableReference(output.Expr); ok {
 			k := tfsandbox.PulumiTopLevelKey(referencedVariableName)
-			inferredType = inferredModuleSchema.Inputs[k].TypeSpec
+			tfName := string(k)
+			pulumiInputName := resource.PropertyKey(strings.ReplaceAll(tfName, "-", "_"))
+			inferredType = inferredModuleSchema.Inputs[pulumiInputName].TypeSpec
 		} else {
 			inferredType = inferExpressionType(output.Expr)
 		}

pkg/modprovider/tfmodules_test.go

@@ -405,6 +405,72 @@ func TestInferringInputsFromLocalPath(t *testing.T) {
 	}
 }
 
+func TestInferringSchemaWithDashedFielsFromLocalPath(t *testing.T) {
+	ctx := context.Background()
+	src := filepath.Join("..", "..", "tests", "testdata", "modules", "dashed-module-fields")
+	p, err := filepath.Abs(src)
+	require.NoError(t, err)
+	for _, executor := range []string{"terraforn", "opentofu"} {
+		logger := newTestLogger(t)
+		t.Run("executor="+executor, func(t *testing.T) {
+			tf := newTestRuntime(t, executor)
+			assert.NoError(t, err, "failed to pick module runtime")
+			inferredSchema, err := inferModuleSchema(ctx, tf,
+				packageName("dashed"),
+				TFModuleSource(p),
+				TFModuleVersion(""),
+				logger)
+			require.NoError(t, err)
+			require.NotNil(t, inferredSchema, "module schema should not be nil")
+
+			expectedInputs := map[resource.PropertyKey]*schema.PropertySpec{
+				"dashed_input": {
+					TypeSpec: stringType,
+				},
+			}
+
+			// Pulumi -> Terraform field name conversion
+			expectedInputFieldMappings := map[resource.PropertyKey]resource.PropertyKey{
+				"dashed_input": "dashed-input",
+			}
+
+			assert.Equal(t, expectedInputs, inferredSchema.Inputs, "inferred inputs do not match")
+			assert.Equal(t, expectedInputFieldMappings, inferredSchema.SchemaFieldMappings.InputFieldMappings)
+
+			expectedOutputs := map[resource.PropertyKey]*schema.PropertySpec{
+				"dashed_output": {
+					TypeSpec: stringType,
+				},
+			}
+
+			// Pulumi -> Terraform field name conversion
+			expectedOutputFieldMappings := map[resource.PropertyKey]resource.PropertyKey{
+				"dashed_output": "dashed-output",
+			}
+
+			assert.Equal(t, expectedOutputs, inferredSchema.Outputs, "inferred outputs do not match")
+			assert.Equal(t, expectedOutputFieldMappings, inferredSchema.SchemaFieldMappings.OutputFieldMappings)
+
+			expectedProviderConfig := schema.ConfigSpec{
+				Variables: map[string]schema.PropertySpec{
+					"google_beta": {
+						Description: "provider configuration for google_beta",
+						TypeSpec:    mapType(anyType),
+					},
+				},
+			}
+
+			expectedProviderFieldMappings := map[string]string{
+				"google_beta": "google-beta",
+			}
+
+			assert.Equal(t, expectedProviderConfig, inferredSchema.ProvidersConfig, "inferred provider config does not match")
+			assert.Equal(t, expectedProviderFieldMappings, inferredSchema.SchemaFieldMappings.ProviderFieldMappings,
+				"inferred provider field mappings do not match")
+		})
+	}
+}
+
 func TestInferModuleSchemaFromGitHubSource(t *testing.T) {
 	ctx := context.Background()
 	packageName := packageName("demoWebsite")

tests/acc_test.go

@@ -1182,6 +1182,38 @@ func TestE2eYAML(t *testing.T) {
 	}
 }
 
+func TestEndToEndUsingModuleWithDashes(t *testing.T) {
+	localProviderBinPath := ensureCompiledProvider(t)
+	localPath := opttest.LocalProviderPath("terraform-module", filepath.Dir(localProviderBinPath))
+	modulePath, err := filepath.Abs(filepath.Join("testdata", "modules", "dashed-module-fields"))
+	assert.NoError(t, err, "failed to get absolute path for dashed module")
+	// TODO[pulumi/pulumi-terraform-module#426] - test local modules with Go
+	for _, runtime := range []string{"yaml", "ts", "python", "dotnet"} {
+		t.Run(runtime, func(t *testing.T) {
+			testProgram, err := filepath.Abs(filepath.Join("testdata", "programs", runtime, "dashed_module"))
+			require.NoError(t, err, "failed to get absolute path for dashed_module program")
+
+			// debug generated Terraform files
+			// tfFilesOutputDir := filepath.Join(testProgram, "tf_files")
+			// t.Cleanup(func() {
+			// 	t.Log("Cleaning up Terraform artifacts")
+			// 	cleanRandomDataFromTerraformArtifacts(t, tfFilesOutputDir, map[string]string{
+			// 		modulePath: "./path/to/module",
+			// 	})
+			// })
+			// add this option to the test program
+			// opttest.Env("PULUMI_TERRAFORM_MODULE_WRITE_TF_FILE", tfFilesOutputDir)
+
+			it := newPulumiTest(t, testProgram, localPath, opttest.SkipInstall())
+			pulumiPackageAdd(t, it, localProviderBinPath, modulePath, "", "dashed")
+			upResult := it.Up(t)
+			result, ok := upResult.Outputs["result"]
+			assert.True(t, ok, "expected output 'result' to be present")
+			assert.Equal(t, "example", result.Value)
+		})
+	}
+}
+
 func TestDiffDetailTerraform(t *testing.T) {
 	w := newTestWriter(t)
 

tests/testdata/modules/dashed-module-fields/main.tf

@@ -0,0 +1,17 @@
+terraform {
+  required_providers {
+    google-beta = {
+      source  = "hashicorp/google-beta"
+      version = ">= 6.19, < 7"
+    }
+  }
+}
+
+variable "dashed-input" {
+    type = string
+    default = "default-value"
+}
+
+output "dashed-output" {
+    value = var.dashed-input
+}