Check example vars

mmv1/api/resource.go

@@ -13,18 +13,23 @@
 package api
 
 import (
+	"bytes"
 	"fmt"
 	"log"
 	"maps"
+	"path/filepath"
 	"regexp"
+	"slices"
 	"sort"
 	"strings"
+	"text/template"
+
+	"github.com/golang/glog"
 
 	"github.com/GoogleCloudPlatform/magic-modules/mmv1/api/product"
 	"github.com/GoogleCloudPlatform/magic-modules/mmv1/api/resource"
 	"github.com/GoogleCloudPlatform/magic-modules/mmv1/api/utils"
 	"github.com/GoogleCloudPlatform/magic-modules/mmv1/google"
-	"golang.org/x/exp/slices"
 )
 
 const RELATIVE_MAGICIAN_LOCATION = "mmv1/"
@@ -1590,13 +1595,45 @@ func (r Resource) FormatDocDescription(desc string, indent bool) string {
 }
 
 func (r Resource) CustomTemplate(templatePath string, appendNewline bool) string {
-	output := resource.ExecuteTemplate(&r, templatePath, appendNewline)
+	output := ExecuteTemplate(&r, templatePath, appendNewline)
 	if !appendNewline {
 		output = strings.TrimSuffix(output, "\n")
 	}
 	return output
 }
 
+func ExecuteTemplate(e any, templatePath string, appendNewline bool) string {
+	templates := []string{
+		templatePath,
+		"templates/terraform/expand_resource_ref.tmpl",
+		"templates/terraform/custom_flatten/bigquery_table_ref.go.tmpl",
+		"templates/terraform/flatten_property_method.go.tmpl",
+		"templates/terraform/expand_property_method.go.tmpl",
+		"templates/terraform/update_mask.go.tmpl",
+		"templates/terraform/nested_query.go.tmpl",
+		"templates/terraform/unordered_list_customize_diff.go.tmpl",
+	}
+	templateFileName := filepath.Base(templatePath)
+
+	tmpl, err := template.New(templateFileName).Funcs(google.TemplateFunctions).ParseFiles(templates...)
+	if err != nil {
+		glog.Exit(err)
+	}
+
+	contents := bytes.Buffer{}
+	if err = tmpl.ExecuteTemplate(&contents, templateFileName, e); err != nil {
+		glog.Exit(err)
+	}
+
+	rs := contents.String()
+
+	if !strings.HasSuffix(rs, "\n") && appendNewline {
+		rs = fmt.Sprintf("%s\n", rs)
+	}
+
+	return rs
+}
+
 // Returns the key of the list of resources in the List API response
 // Used to get the list of resources to sweep
 func (r Resource) ResourceListKey() string {

mmv1/api/resource/examples.go

@@ -18,6 +18,7 @@ import (
 	"fmt"
 	"log"
 	"net/url"
+	"os"
 	"path/filepath"
 	"regexp"
 	"slices"
@@ -201,6 +202,22 @@ func (e *Examples) Validate(rName string) {
 	e.ValidateExternalProviders()
 }
 
+func validateRegexForContents(r *regexp.Regexp, contents string, configPath string, objName string, vars map[string]string) {
+	matches := r.FindAllStringSubmatch(contents, -1)
+	for _, v := range matches {
+		found := false
+		for k, _ := range vars {
+			if k == v[1] {
+				found = true
+				break
+			}
+		}
+		if !found {
+			log.Fatalf("Failed to find %s environment variable defined in YAML file when validating the file %s. Please define this in %s", v[1], configPath, objName)
+		}
+	}
+}
+
 func (e *Examples) ValidateExternalProviders() {
 	// Official providers supported by HashiCorp
 	// https://registry.terraform.io/search/providers?namespace=hashicorp&tier=official
@@ -249,7 +266,7 @@ func (e *Examples) SetHCLText() {
 		docTestEnvVars[key] = docs_defaults[e.TestEnvVars[key]]
 	}
 	e.TestEnvVars = docTestEnvVars
-	e.DocumentationHCLText = ExecuteTemplate(e, e.ConfigPath, true)
+	e.DocumentationHCLText = e.ExecuteTemplate()
 	e.DocumentationHCLText = regexp.MustCompile(`\n\n$`).ReplaceAllString(e.DocumentationHCLText, "\n")
 
 	// Remove region tags
@@ -290,7 +307,7 @@ func (e *Examples) SetHCLText() {
 
 	e.Vars = testVars
 	e.TestEnvVars = testTestEnvVars
-	e.TestHCLText = ExecuteTemplate(e, e.ConfigPath, true)
+	e.TestHCLText = e.ExecuteTemplate()
 	e.TestHCLText = regexp.MustCompile(`\n\n$`).ReplaceAllString(e.TestHCLText, "\n")
 	// Remove region tags
 	e.TestHCLText = re1.ReplaceAllString(e.TestHCLText, "")
@@ -302,20 +319,23 @@ func (e *Examples) SetHCLText() {
 	e.TestEnvVars = originalTestEnvVars
 }
 
-func ExecuteTemplate(e any, templatePath string, appendNewline bool) string {
-	templates := []string{
-		templatePath,
-		"templates/terraform/expand_resource_ref.tmpl",
-		"templates/terraform/custom_flatten/bigquery_table_ref.go.tmpl",
-		"templates/terraform/flatten_property_method.go.tmpl",
-		"templates/terraform/expand_property_method.go.tmpl",
-		"templates/terraform/update_mask.go.tmpl",
-		"templates/terraform/nested_query.go.tmpl",
-		"templates/terraform/unordered_list_customize_diff.go.tmpl",
+func (e *Examples) ExecuteTemplate() string {
+	templateContent, err := os.ReadFile(e.ConfigPath)
+	if err != nil {
+		glog.Exit(err)
 	}
-	templateFileName := filepath.Base(templatePath)
 
-	tmpl, err := template.New(templateFileName).Funcs(google.TemplateFunctions).ParseFiles(templates...)
+	fileContentString := string(templateContent)
+
+	// Check that any variables in Vars or TestEnvVars used in the example are defined via YAML
+	envVarRegex := regexp.MustCompile(`{{index \$\.TestEnvVars "([a-zA-Z_]*)"}}`)
+	validateRegexForContents(envVarRegex, fileContentString, e.ConfigPath, "test_env_vars", e.TestEnvVars)
+	varRegex := regexp.MustCompile(`{{index \$\.Vars "([a-zA-Z_]*)"}}`)
+	validateRegexForContents(varRegex, fileContentString, e.ConfigPath, "vars", e.Vars)
+
+	templateFileName := filepath.Base(e.ConfigPath)
+
+	tmpl, err := template.New(templateFileName).Funcs(google.TemplateFunctions).Parse(fileContentString)
 	if err != nil {
 		glog.Exit(err)
 	}
@@ -327,7 +347,7 @@ func ExecuteTemplate(e any, templatePath string, appendNewline bool) string {
 
 	rs := contents.String()
 
-	if !strings.HasSuffix(rs, "\n") && appendNewline {
+	if !strings.HasSuffix(rs, "\n") {
 		rs = fmt.Sprintf("%s\n", rs)
 	}
 
@@ -401,7 +421,7 @@ func (e *Examples) SetOiCSHCLText() {
 	}
 
 	e.Vars = testVars
-	e.OicsHCLText = ExecuteTemplate(e, e.ConfigPath, true)
+	e.OicsHCLText = e.ExecuteTemplate()
 	e.OicsHCLText = regexp.MustCompile(`\n\n$`).ReplaceAllString(e.OicsHCLText, "\n")
 
 	// Remove region tags

mmv1/api/type.go

@@ -1078,7 +1078,7 @@ func (t Type) NamespaceProperty() string {
 }
 
 func (t Type) CustomTemplate(templatePath string, appendNewline bool) string {
-	return resource.ExecuteTemplate(&t, templatePath, appendNewline)
+	return ExecuteTemplate(&t, templatePath, appendNewline)
 }
 
 func (t *Type) GetIdFormat() string {

mmv1/products/accesscontextmanager/AuthorizedOrgsDesc.yaml

@@ -61,6 +61,8 @@ examples:
   - name: 'access_context_manager_authorized_orgs_desc_basic'
     primary_resource_id: 'authorized-orgs-desc'
     exclude_test: true
+    test_env_vars:
+      org_id: 'ORG_ID'
 parameters:
   - name: 'parent'
     type: String

mmv1/products/apigee/EnvgroupAttachment.yaml

@@ -52,6 +52,9 @@ examples:
       project_id: 'my-project'
       envgroup_name: 'my-envgroup'
       environment_name: 'my-environment'
+    test_env_vars:
+      org_id: 'ORG_ID'
+      billing_account: 'BILLING_ACCT'
     exclude_test: true
   - name: 'apigee_environment_group_attachment_basic_test'
     primary_resource_id: 'apigee_environment_group_attachment'

mmv1/products/apigee/InstanceAttachment.yaml

@@ -51,6 +51,9 @@ examples:
       project_id: 'my-project'
       instance_name: 'my-instance-name'
       environment_name: 'my-environment-name'
+    test_env_vars:
+      org_id: 'ORG_ID'
+      billing_account: 'BILLING_ACCT'
     exclude_test: true
       # This is a more verbose version of the above that creates all
       # the resources needed for the acceptance test.

mmv1/products/bigquerydatatransfer/Config.yaml

@@ -60,6 +60,7 @@ examples:
       dataset_id: 'example_dataset'
       key_name: 'example-key'
       keyring_name: 'example-keyring'
+      display_name: 'display-name'
     exclude_test: true
   - name: 'bigquerydatatransfer_config_salesforce'
     primary_resource_id: 'salesforce_config'

mmv1/products/cloudbuild/Trigger.yaml

@@ -82,6 +82,8 @@ examples:
       cloudbuild_trigger_name: 'manual-trigger'
   - name: 'cloudbuild_trigger_manual_github_enterprise'
     primary_resource_id: 'manual-ghe-trigger'
+    vars:
+      cloudbuild_trigger_name: 'my-trigger'
     exclude_test: true
   - name: 'cloudbuild_trigger_manual_bitbucket_server'
     primary_resource_id: 'manual-bitbucket-trigger'

mmv1/products/colab/Schedule.yaml

@@ -76,6 +76,7 @@ examples:
       dataform_repository: 'dataform-repository'
       start_time: '2014-10-02T15:01:23Z'
       end_time: '2014-10-10T15:01:23Z'
+      key_name: 'my-key'
     test_env_vars:
       project_id: 'PROJECT_NAME'
       location: 'REGION'

mmv1/products/datastream/Stream.yaml

@@ -216,6 +216,7 @@ examples:
   - name: 'datastream_stream_salesforce'
     primary_resource_id: 'default'
     vars:
+      stream_id: 'sf-stream'
       source_connection_profile_id: 'source-profile'
       destination_connection_profile_id: 'destination-profile'
     exclude_test: true

mmv1/products/dialogflow/EntityType.yaml

@@ -38,7 +38,7 @@ examples:
   - name: 'dialogflow_entity_type_basic'
     primary_resource_id: 'basic_entity_type'
     vars:
-      intent_name: 'basic-entity-type'
+      entity_type_name: 'basic-entity-type'
     exclude_test: true
 parameters:
 properties:

mmv1/products/eventarc/Trigger.yaml

@@ -48,6 +48,7 @@ examples:
     primary_resource_id: primary
     vars:
       trigger_name: some-trigger
+      network_attachment_name: network-attachment
     test_vars_overrides:
       'network_attachment_name': 'acctest.BootstrapNetworkAttachment(t, "tf-test-eventarc-trigger-na", acctest.BootstrapSubnet(t, "tf-test-eventarc-trigger-subnet", acctest.BootstrapSharedTestNetwork(t, "tf-test-eventarc-trigger-network")))'
     test_env_vars:

mmv1/products/firebaseapphosting/DefaultDomain.yaml

@@ -33,6 +33,7 @@ examples:
     primary_resource_id: example
     vars:
       backend_id: 'dd-mini'
+      service_act_id: 'service-account'
     test_env_vars:
       project_id: 'PROJECT_NAME'
     test_vars_overrides:
@@ -42,6 +43,7 @@ examples:
     primary_resource_id: example
     vars:
       backend_id: 'dd-full'
+      service_act_id: 'service-account'
     test_env_vars:
       project_id: 'PROJECT_NAME'
     test_vars_overrides:
@@ -51,6 +53,7 @@ examples:
     primary_resource_id: example
     vars:
       backend_id: 'dd-disabled'
+      service_act_id: 'service-account'
     test_env_vars:
       project_id: 'PROJECT_NAME'
     test_vars_overrides:

mmv1/products/firebaseapphosting/Domain.yaml

@@ -45,6 +45,8 @@ examples:
     primary_resource_id: example
     vars:
       backend_id: 'domain-mini'
+      service_act_id: 'sa-id'
+      domain_id: example.com
     test_env_vars:
       project_id: 'PROJECT_NAME'
     test_vars_overrides:
@@ -55,6 +57,8 @@ examples:
     primary_resource_id: example
     vars:
       backend_id: 'domain-full'
+      service_act_id: 'sa-id'
+      domain_id: example.com
     test_env_vars:
       project_id: 'PROJECT_NAME'
     test_vars_overrides:

mmv1/products/firebaseextensions/Instance.yaml

@@ -52,6 +52,7 @@ examples:
       instance-id: 'storage-resize-images'
       bucket_id: 'bucket-id'
       service-account-id: 's-a'
+      location: "us-central1"
     test_env_vars:
       project_id: 'PROJECT_NAME'
     test_vars_overrides:

mmv1/products/netapp/Backup.yaml

@@ -68,6 +68,7 @@ examples:
       volume_name: 'backup-volume'
       backup_vault_name: 'backup-vault'
       backup_name: 'test-backup'
+      network_name: 'network'
     test_vars_overrides:
       'network_name': 'acctest.BootstrapSharedServiceNetworkingConnection(t, "gcnv-network-config-2", acctest.ServiceNetworkWithParentService("netapp.servicenetworking.goog"))'
 parameters:

mmv1/products/securitycenter/OrganizationSccBigQueryExport.yaml

@@ -45,7 +45,7 @@ examples:
     primary_resource_id: 'custom_big_query_export_config'
     vars:
       big_query_export_id: 'my-export'
-      dataset: 'my-dataset'
+      dataset_id: 'my-dataset'
       name: 'my-export'
     test_env_vars:
       org_id: 'ORG_ID'

mmv1/products/securitycenterv2/OrganizationSccBigQueryExport.yaml

@@ -46,7 +46,7 @@ examples:
     primary_resource_id: 'custom_big_query_export_config'
     vars:
       big_query_export_id: 'my-export'
-      dataset: 'my-dataset'
+      dataset_id: 'my-dataset'
       name: 'my-export'
     test_env_vars:
       org_id: 'ORG_ID'

mmv1/products/securitycenterv2/OrganizationSccBigQueryExports.yaml

@@ -47,7 +47,7 @@ examples:
     primary_resource_id: 'custom_big_query_export_config'
     vars:
       big_query_export_id: 'my-export'
-      dataset: 'my-dataset'
+      dataset_id: 'my-dataset'
       name: 'my-export'
     test_env_vars:
       org_id: 'ORG_ID'

mmv1/products/vmwareengine/ExternalAddress.yaml

@@ -62,6 +62,8 @@ examples:
       private_cloud_id: 'sample-pc'
       management_cluster_id: 'sample-mgmt-cluster'
       network_policy_id: 'sample-np'
+    test_env_vars:
+      region: 'REGION'
  # update tests will take care of all CRUD tests. Parent PC creation is expensive and node reservation is required.
     exclude_test: true
 parameters:

mmv1/templates/terraform/examples/bigquery_analyticshub_listing_subscription_basic.tf.tmpl

@@ -2,15 +2,15 @@ resource "google_bigquery_analytics_hub_data_exchange" "{{$.PrimaryResourceId}}"
   location         = "US"
   data_exchange_id = "{{index $.Vars "data_exchange_id"}}"
   display_name     = "{{index $.Vars "data_exchange_id"}}"
-  description      = "{{index $.Vars "desc"}}"
+  description      = "Test Description"
 }
 
 resource "google_bigquery_analytics_hub_listing" "{{$.PrimaryResourceId}}" {
   location         = "US"
   data_exchange_id = google_bigquery_analytics_hub_data_exchange.{{$.PrimaryResourceId}}.data_exchange_id
   listing_id       = "{{index $.Vars "listing_id"}}"
   display_name     = "{{index $.Vars "listing_id"}}"
-  description      = "{{index $.Vars "desc"}}"
+  description      = "Test Description"
 
   bigquery_dataset {
     dataset = google_bigquery_dataset.{{$.PrimaryResourceId}}.id
@@ -20,7 +20,7 @@ resource "google_bigquery_analytics_hub_listing" "{{$.PrimaryResourceId}}" {
 resource "google_bigquery_dataset" "{{$.PrimaryResourceId}}" {
   dataset_id                  = "{{index $.Vars "listing_id"}}"
   friendly_name               = "{{index $.Vars "listing_id"}}"
-  description                 = "{{index $.Vars "desc"}}"
+  description                 = "Test Description"
   location                    = "US"
 }
 

mmv1/templates/terraform/examples/edgecontainer_local_control_plane_node_pool.tf.tmpl

@@ -1,5 +1,5 @@
 resource "google_edgecontainer_cluster" "{{$.PrimaryResourceId}}" {
-  name = "{{index $.Vars "edgecontainer_cluster_name"}}"
+  name = "default"
   location = "us-central1"
 
   authorization {

mmv1/templates/terraform/examples/gkebackup_backupchannel_basic.tf.tmpl

@@ -1,7 +1,7 @@
 resource "google_gke_backup_backup_channel" "basic" {
   name = "{{index $.Vars "name"}}"
   location = "us-central1"
-  description = "{{index $.Vars "description"}}"
+  description = "Description"
   destination_project = "{{index $.Vars "destination_project"}}"
   labels = { "key": "some-value" }
 }