autogeneration of tests for singular datasources (#14872)

Co-authored-by: Stephen Lewis (Burrows) <[email protected]>

Author: luckyswaminathan

mmv1/api/resource.go

@@ -1203,31 +1203,42 @@ func ImportIdFormats(importFormat, identity []string, baseUrl string) []string {
 	return uniq
 }
 
-func (r Resource) IgnoreReadPropertiesToString(e resource.Examples) string {
+// IgnoreReadProperties returns a sorted slice of property names (snake_case) that should be ignored when reading.
+// This is useful for downstream code that needs to iterate over these properties.
+func (r Resource) IgnoreReadProperties(e resource.Examples) []string {
 	var props []string
 	for _, tp := range r.AllUserProperties() {
 		if tp.UrlParamOnly || tp.IsA("ResourceRef") {
-			props = append(props, fmt.Sprintf("\"%s\"", google.Underscore(tp.Name)))
+			props = append(props, google.Underscore(tp.Name))
 		}
 	}
-	for _, tp := range e.IgnoreReadExtra {
-		props = append(props, fmt.Sprintf("\"%s\"", tp))
-	}
-	for _, tp := range r.IgnoreReadLabelsFields(r.PropertiesWithExcluded()) {
-		props = append(props, fmt.Sprintf("\"%s\"", tp))
-	}
-	for _, tp := range ignoreReadFields(r.AllUserProperties()) {
-		props = append(props, fmt.Sprintf("\"%s\"", tp))
-	}
+	props = append(props, e.IgnoreReadExtra...)
+	props = append(props, r.IgnoreReadLabelsFields(r.PropertiesWithExcluded())...)
+	props = append(props, ignoreReadFields(r.AllUserProperties())...)
 
 	slices.Sort(props)
+	return props
+}
 
+// IgnoreReadPropertiesToString returns the ignore read properties as a Go-syntax string slice.
+// This is a wrapper around IgnoreReadProperties for backwards compatibility.
+func (r Resource) IgnoreReadPropertiesToString(e resource.Examples) string {
+	props := r.IgnoreReadProperties(e)
 	if len(props) > 0 {
-		return fmt.Sprintf("[]string{%s}", strings.Join(props, ", "))
+		return fmt.Sprintf("[]string{%s}", strings.Join(quoteStrings(props), ", "))
 	}
 	return ""
 }
 
+// quoteStrings returns a new slice with each string quoted.
+func quoteStrings(strs []string) []string {
+	quoted := make([]string, len(strs))
+	for i, s := range strs {
+		quoted[i] = fmt.Sprintf("\"%s\"", s)
+	}
+	return quoted
+}
+
 func ignoreReadFields(props []*Type) []string {
 	var fields []string
 	for _, tp := range props {
@@ -2021,6 +2032,13 @@ func (r *Resource) ShouldGenerateSingularDataSource() bool {
 	return r.Datasource.Generate
 }
 
+func (r *Resource) ShouldGenerateSingularDataSourceTests() bool {
+	if r.Datasource == nil {
+		return false
+	}
+	return !r.Datasource.ExcludeTest
+}
+
 func (r Resource) ShouldDatasourceSetLabels() bool {
 	for _, p := range r.Properties {
 		if p.Name == "labels" && p.Type == "KeyValueLabels" {

mmv1/api/resource/datasource.go

@@ -16,4 +16,6 @@ package resource
 type Datasource struct {
 	// boolean to determine whether the datasource file should be generated
 	Generate bool `yaml:"generate"`
+	// boolean to determine whether tests should be generated for a datasource
+	ExcludeTest bool `yaml:"exclude_test"`
 }

mmv1/products/cloudrun/Service.yaml

@@ -31,6 +31,7 @@ import_format:
   - 'locations/{{location}}/namespaces/{{project}}/services/{{name}}'
 datasource:
   generate: true
+  exclude_test: true
 timeouts:
   insert_minutes: 20
   update_minutes: 20

mmv1/products/storagecontrol/FolderIntelligenceConfig.yaml

@@ -51,6 +51,7 @@ autogen_async: false
 
 datasource:
   generate: true
+  exclude_test: true
 
 examples:
   - name: storage_control_folder_intelligence_config_basic

mmv1/products/storagecontrol/OrganizationIntelligenceConfig.yaml

@@ -51,6 +51,7 @@ autogen_async: false
 
 datasource:
   generate: true
+  exclude_test: true
 
 examples:
   - name: storage_control_organization_intelligence_config_basic

mmv1/products/storagecontrol/ProjectIntelligenceConfig.yaml

@@ -52,6 +52,7 @@ autogen_async: false
 
 datasource:
   generate: true
+  exclude_test: true
 
 examples:
   - name: storage_control_project_intelligence_config_basic

mmv1/provider/template_data.go

@@ -165,6 +165,35 @@ func (td *TemplateData) GenerateTestFile(filePath string, resource api.Resource)
 	td.GenerateFile(filePath, templatePath, tmplInput, true, templates...)
 }
 
+func (td *TemplateData) GenerateDataSourceTestFile(filePath string, resource api.Resource) {
+	templatePath := "templates/terraform/examples/base_configs/datasource_test_file.go.tmpl"
+	templates := []string{
+		"templates/terraform/env_var_context.go.tmpl",
+		templatePath,
+	}
+	tmplInput := TestInput{
+		Res:                  resource,
+		ImportPath:           resource.ImportPath,
+		PROJECT_NAME:         "my-project-name",
+		CREDENTIALS:          "my/credentials/filename.json",
+		REGION:               "us-west1",
+		ORG_ID:               "123456789",
+		ORG_DOMAIN:           "example.com",
+		ORG_TARGET:           "123456789",
+		PROJECT_NUMBER:       "1111111111111",
+		BILLING_ACCT:         "000000-0000000-0000000-000000",
+		MASTER_BILLING_ACCT:  "000000-0000000-0000000-000000",
+		SERVICE_ACCT:         "[email protected]",
+		CUST_ID:              "A01b123xz",
+		IDENTITY_USER:        "cloud_identity_user",
+		PAP_DESCRIPTION:      "description",
+		CHRONICLE_ID:         "00000000-0000-0000-0000-000000000000",
+		VMWAREENGINE_PROJECT: "my-vmwareengine-project",
+	}
+
+	td.GenerateFile(filePath, templatePath, tmplInput, true, templates...)
+}
+
 func (td *TemplateData) GenerateIamPolicyFile(filePath string, resource api.Resource) {
 	templatePath := "templates/terraform/iam_policy.go.tmpl"
 	templates := []string{

mmv1/provider/terraform.go

@@ -108,6 +108,7 @@ func (t *Terraform) GenerateObject(object api.Resource, outputFolder, productPat
 			t.GenerateResourceTests(object, *templateData, outputFolder)
 			t.GenerateResourceSweeper(object, *templateData, outputFolder)
 			t.GenerateSingularDataSource(object, *templateData, outputFolder)
+			t.GenerateSingularDataSourceTests(object, *templateData, outputFolder)
 			// log.Printf("Generating %s metadata", object.Name)
 			t.GenerateResourceMetadata(object, *templateData, outputFolder)
 		}
@@ -208,6 +209,21 @@ func (t *Terraform) GenerateSingularDataSource(object api.Resource, templateData
 	templateData.GenerateDataSourceFile(targetFilePath, object)
 }
 
+func (t *Terraform) GenerateSingularDataSourceTests(object api.Resource, templateData TemplateData, outputFolder string) {
+	if !object.ShouldGenerateSingularDataSourceTests() {
+		return
+	}
+
+	productName := t.Product.ApiName
+	targetFolder := path.Join(outputFolder, t.FolderName(), "services", productName)
+	if err := os.MkdirAll(targetFolder, os.ModePerm); err != nil {
+		log.Println(fmt.Errorf("error creating parent directory %v: %v", targetFolder, err))
+	}
+	targetFilePath := path.Join(targetFolder, fmt.Sprintf("data_source_%s_test.go", t.ResourceGoFilename(object)))
+	templateData.GenerateDataSourceTestFile(targetFilePath, object)
+
+}
+
 // GenerateProduct creates the product.go file for a given service directory.
 // This will be used to seed the directory and add a package-level comment
 // specific to the product.

mmv1/templates/terraform/examples/base_configs/datasource_test_file.go.tmpl

@@ -0,0 +1,96 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+// ----------------------------------------------------------------------------
+//
+//     *** AUTO GENERATED CODE    *** Type: MMv1     ***
+//
+// ----------------------------------------------------------------------------
+//
+//     This file is automatically generated by Magic Modules and manual
+//     changes will be clobbered when the file is regenerated.
+//
+//     Please read more about how to change this file in
+//     .github/CONTRIBUTING.md.
+//
+// ----------------------------------------------------------------------------
+
+package {{ $.Res.PackageName }}_test
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/hashicorp/terraform-plugin-testing/helper/resource"
+
+	"{{ $.ImportPath }}/acctest"
+	"{{ $.ImportPath }}/envvar"
+	"{{ $.ImportPath }}/tpgresource"
+)
+
+{{ if $.Res.TestExamples }}
+{{ $e := index $.Res.TestExamples 0 }}
+func TestAccDataSource{{ $.Res.ResourceName }}_basic(t *testing.T) {
+	t.Parallel()
+
+	context := map[string]interface{}{
+	{{- template "EnvVarContext" dict "TestEnvVars" $e.TestEnvVars "HasNewLine" false}}
+	{{- range $varKey, $varVal := $e.TestVarsOverrides }}
+		"{{$varKey}}": {{$varVal}},
+	{{- end }}
+		"random_suffix": acctest.RandString(t, 10),
+	}
+
+	acctest.VcrTest(t, resource.TestCase{
+		PreCheck:                 func() { acctest.AccTestPreCheck(t) },
+	{{- if $.Res.VersionedProvider $e.MinVersion }}
+		ProtoV5ProviderFactories: acctest.ProtoV5ProviderBetaFactories(t),
+	{{- else }}
+		ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t),
+	{{- end }}
+	{{- if $e.ExternalProviders }}
+		ExternalProviders: map[string]resource.ExternalProvider{
+		{{- range $provider := $e.ExternalProviders }}
+			"{{$provider}}": {},
+		{{- end }}
+		},
+	{{- end }}
+	{{- if not $.Res.ExcludeDelete }}
+		CheckDestroy: testAccCheck{{ $.Res.ResourceName }}DestroyProducer(t),
+{{- end }}
+		Steps: []resource.TestStep{
+			{
+				Config: testAcc{{ $e.TestSlug $.Res.ProductMetadata.Name $.Res.Name }}DataSource(context),
+				Check: resource.ComposeTestCheckFunc(
+					{{- if gt (len ($.Res.IgnoreReadProperties $e)) 0 }}
+						acctest.CheckDataSourceStateMatchesResourceStateWithIgnores(
+							"data.{{ $e.ResourceType $.Res.TerraformName }}.default",
+							"{{ $e.ResourceType $.Res.TerraformName }}.{{ $e.PrimaryResourceId }}",
+							map[string]struct{}{
+								{{- range $prop := $.Res.IgnoreReadProperties $e }}
+								"{{ $prop }}": {},
+								{{- end }}
+							},
+						),
+					{{- else }}
+										acctest.CheckDataSourceStateMatchesResourceState("data.{{ $e.ResourceType $.Res.TerraformName }}.default", "{{ $e.ResourceType $.Res.TerraformName }}.{{ $e.PrimaryResourceId }}"),
+					{{- end }}
+					),
+			},
+		},
+	})
+}
+
+func testAcc{{ $e.TestSlug $.Res.ProductMetadata.Name $.Res.Name }}DataSource(context map[string]interface{}) string {
+	return acctest.Nprintf(`
+{{ $e.TestHCLText }}
+
+data "{{ $.Res.TerraformName }}" "default" {
+{{- range $fieldName := $.Res.DatasourceRequiredFields }}
+  {{ $fieldName }} = {{ $e.ResourceType $.Res.TerraformName }}.{{ $e.PrimaryResourceId }}.{{ $fieldName }}
+{{- end }}
+}`,
+		context,
+	)
+}
+{{ end }}