diff --git a/mmv1/api/resource.go b/mmv1/api/resource.go index 42b2315c2d14..2ae709dc0d81 100644 --- a/mmv1/api/resource.go +++ b/mmv1/api/resource.go @@ -13,18 +13,23 @@ package api import ( + "bytes" "fmt" "log" "maps" + "path/filepath" "regexp" + "slices" "sort" "strings" + "text/template" + + "github.com/golang/glog" "github.com/GoogleCloudPlatform/magic-modules/mmv1/api/product" "github.com/GoogleCloudPlatform/magic-modules/mmv1/api/resource" "github.com/GoogleCloudPlatform/magic-modules/mmv1/api/utils" "github.com/GoogleCloudPlatform/magic-modules/mmv1/google" - "golang.org/x/exp/slices" ) const RELATIVE_MAGICIAN_LOCATION = "mmv1/" @@ -1590,13 +1595,45 @@ func (r Resource) FormatDocDescription(desc string, indent bool) string { } func (r Resource) CustomTemplate(templatePath string, appendNewline bool) string { - output := resource.ExecuteTemplate(&r, templatePath, appendNewline) + output := ExecuteTemplate(&r, templatePath, appendNewline) if !appendNewline { output = strings.TrimSuffix(output, "\n") } return output } +func ExecuteTemplate(e any, templatePath string, appendNewline bool) string { + templates := []string{ + templatePath, + "templates/terraform/expand_resource_ref.tmpl", + "templates/terraform/custom_flatten/bigquery_table_ref.go.tmpl", + "templates/terraform/flatten_property_method.go.tmpl", + "templates/terraform/expand_property_method.go.tmpl", + "templates/terraform/update_mask.go.tmpl", + "templates/terraform/nested_query.go.tmpl", + "templates/terraform/unordered_list_customize_diff.go.tmpl", + } + templateFileName := filepath.Base(templatePath) + + tmpl, err := template.New(templateFileName).Funcs(google.TemplateFunctions).ParseFiles(templates...) + if err != nil { + glog.Exit(err) + } + + contents := bytes.Buffer{} + if err = tmpl.ExecuteTemplate(&contents, templateFileName, e); err != nil { + glog.Exit(err) + } + + rs := contents.String() + + if !strings.HasSuffix(rs, "\n") && appendNewline { + rs = fmt.Sprintf("%s\n", rs) + } + + return rs +} + // Returns the key of the list of resources in the List API response // Used to get the list of resources to sweep func (r Resource) ResourceListKey() string { diff --git a/mmv1/api/resource/examples.go b/mmv1/api/resource/examples.go index b2d267363e8d..89a981d9f329 100644 --- a/mmv1/api/resource/examples.go +++ b/mmv1/api/resource/examples.go @@ -18,6 +18,7 @@ import ( "fmt" "log" "net/url" + "os" "path/filepath" "regexp" "slices" @@ -201,6 +202,22 @@ func (e *Examples) Validate(rName string) { e.ValidateExternalProviders() } +func validateRegexForContents(r *regexp.Regexp, contents string, configPath string, objName string, vars map[string]string) { + matches := r.FindAllStringSubmatch(contents, -1) + for _, v := range matches { + found := false + for k, _ := range vars { + if k == v[1] { + found = true + break + } + } + if !found { + log.Fatalf("Failed to find %s environment variable defined in YAML file when validating the file %s. Please define this in %s", v[1], configPath, objName) + } + } +} + func (e *Examples) ValidateExternalProviders() { // Official providers supported by HashiCorp // https://registry.terraform.io/search/providers?namespace=hashicorp&tier=official @@ -249,7 +266,7 @@ func (e *Examples) SetHCLText() { docTestEnvVars[key] = docs_defaults[e.TestEnvVars[key]] } e.TestEnvVars = docTestEnvVars - e.DocumentationHCLText = ExecuteTemplate(e, e.ConfigPath, true) + e.DocumentationHCLText = e.ExecuteTemplate() e.DocumentationHCLText = regexp.MustCompile(`\n\n$`).ReplaceAllString(e.DocumentationHCLText, "\n") // Remove region tags @@ -290,7 +307,7 @@ func (e *Examples) SetHCLText() { e.Vars = testVars e.TestEnvVars = testTestEnvVars - e.TestHCLText = ExecuteTemplate(e, e.ConfigPath, true) + e.TestHCLText = e.ExecuteTemplate() e.TestHCLText = regexp.MustCompile(`\n\n$`).ReplaceAllString(e.TestHCLText, "\n") // Remove region tags e.TestHCLText = re1.ReplaceAllString(e.TestHCLText, "") @@ -302,20 +319,23 @@ func (e *Examples) SetHCLText() { e.TestEnvVars = originalTestEnvVars } -func ExecuteTemplate(e any, templatePath string, appendNewline bool) string { - templates := []string{ - templatePath, - "templates/terraform/expand_resource_ref.tmpl", - "templates/terraform/custom_flatten/bigquery_table_ref.go.tmpl", - "templates/terraform/flatten_property_method.go.tmpl", - "templates/terraform/expand_property_method.go.tmpl", - "templates/terraform/update_mask.go.tmpl", - "templates/terraform/nested_query.go.tmpl", - "templates/terraform/unordered_list_customize_diff.go.tmpl", +func (e *Examples) ExecuteTemplate() string { + templateContent, err := os.ReadFile(e.ConfigPath) + if err != nil { + glog.Exit(err) } - templateFileName := filepath.Base(templatePath) - tmpl, err := template.New(templateFileName).Funcs(google.TemplateFunctions).ParseFiles(templates...) + fileContentString := string(templateContent) + + // Check that any variables in Vars or TestEnvVars used in the example are defined via YAML + envVarRegex := regexp.MustCompile(`{{index \$\.TestEnvVars "([a-zA-Z_]*)"}}`) + validateRegexForContents(envVarRegex, fileContentString, e.ConfigPath, "test_env_vars", e.TestEnvVars) + varRegex := regexp.MustCompile(`{{index \$\.Vars "([a-zA-Z_]*)"}}`) + validateRegexForContents(varRegex, fileContentString, e.ConfigPath, "vars", e.Vars) + + templateFileName := filepath.Base(e.ConfigPath) + + tmpl, err := template.New(templateFileName).Funcs(google.TemplateFunctions).Parse(fileContentString) if err != nil { glog.Exit(err) } @@ -327,7 +347,7 @@ func ExecuteTemplate(e any, templatePath string, appendNewline bool) string { rs := contents.String() - if !strings.HasSuffix(rs, "\n") && appendNewline { + if !strings.HasSuffix(rs, "\n") { rs = fmt.Sprintf("%s\n", rs) } @@ -401,7 +421,7 @@ func (e *Examples) SetOiCSHCLText() { } e.Vars = testVars - e.OicsHCLText = ExecuteTemplate(e, e.ConfigPath, true) + e.OicsHCLText = e.ExecuteTemplate() e.OicsHCLText = regexp.MustCompile(`\n\n$`).ReplaceAllString(e.OicsHCLText, "\n") // Remove region tags diff --git a/mmv1/api/type.go b/mmv1/api/type.go index 6c0d5e15920e..75d41ddcb78f 100644 --- a/mmv1/api/type.go +++ b/mmv1/api/type.go @@ -1078,7 +1078,7 @@ func (t Type) NamespaceProperty() string { } func (t Type) CustomTemplate(templatePath string, appendNewline bool) string { - return resource.ExecuteTemplate(&t, templatePath, appendNewline) + return ExecuteTemplate(&t, templatePath, appendNewline) } func (t *Type) GetIdFormat() string { diff --git a/mmv1/products/accesscontextmanager/AuthorizedOrgsDesc.yaml b/mmv1/products/accesscontextmanager/AuthorizedOrgsDesc.yaml index e8b1cfd103f9..680d5a717fd4 100644 --- a/mmv1/products/accesscontextmanager/AuthorizedOrgsDesc.yaml +++ b/mmv1/products/accesscontextmanager/AuthorizedOrgsDesc.yaml @@ -61,6 +61,8 @@ examples: - name: 'access_context_manager_authorized_orgs_desc_basic' primary_resource_id: 'authorized-orgs-desc' exclude_test: true + test_env_vars: + org_id: 'ORG_ID' parameters: - name: 'parent' type: String diff --git a/mmv1/products/apigee/EnvgroupAttachment.yaml b/mmv1/products/apigee/EnvgroupAttachment.yaml index 3d659175fd38..ac75d8abae38 100644 --- a/mmv1/products/apigee/EnvgroupAttachment.yaml +++ b/mmv1/products/apigee/EnvgroupAttachment.yaml @@ -52,6 +52,9 @@ examples: project_id: 'my-project' envgroup_name: 'my-envgroup' environment_name: 'my-environment' + test_env_vars: + org_id: 'ORG_ID' + billing_account: 'BILLING_ACCT' exclude_test: true - name: 'apigee_environment_group_attachment_basic_test' primary_resource_id: 'apigee_environment_group_attachment' diff --git a/mmv1/products/apigee/InstanceAttachment.yaml b/mmv1/products/apigee/InstanceAttachment.yaml index 1ae86f456e68..942b765976ea 100644 --- a/mmv1/products/apigee/InstanceAttachment.yaml +++ b/mmv1/products/apigee/InstanceAttachment.yaml @@ -51,6 +51,9 @@ examples: project_id: 'my-project' instance_name: 'my-instance-name' environment_name: 'my-environment-name' + test_env_vars: + org_id: 'ORG_ID' + billing_account: 'BILLING_ACCT' exclude_test: true # This is a more verbose version of the above that creates all # the resources needed for the acceptance test. diff --git a/mmv1/products/bigquerydatatransfer/Config.yaml b/mmv1/products/bigquerydatatransfer/Config.yaml index 50628f0c56cb..8206309eee45 100644 --- a/mmv1/products/bigquerydatatransfer/Config.yaml +++ b/mmv1/products/bigquerydatatransfer/Config.yaml @@ -60,6 +60,7 @@ examples: dataset_id: 'example_dataset' key_name: 'example-key' keyring_name: 'example-keyring' + display_name: 'display-name' exclude_test: true - name: 'bigquerydatatransfer_config_salesforce' primary_resource_id: 'salesforce_config' diff --git a/mmv1/products/cloudbuild/Trigger.yaml b/mmv1/products/cloudbuild/Trigger.yaml index db31f044d3f6..f4367b6eab7d 100644 --- a/mmv1/products/cloudbuild/Trigger.yaml +++ b/mmv1/products/cloudbuild/Trigger.yaml @@ -82,6 +82,8 @@ examples: cloudbuild_trigger_name: 'manual-trigger' - name: 'cloudbuild_trigger_manual_github_enterprise' primary_resource_id: 'manual-ghe-trigger' + vars: + cloudbuild_trigger_name: 'my-trigger' exclude_test: true - name: 'cloudbuild_trigger_manual_bitbucket_server' primary_resource_id: 'manual-bitbucket-trigger' diff --git a/mmv1/products/colab/Schedule.yaml b/mmv1/products/colab/Schedule.yaml index 29398afea896..9aa46e282b57 100644 --- a/mmv1/products/colab/Schedule.yaml +++ b/mmv1/products/colab/Schedule.yaml @@ -76,6 +76,7 @@ examples: dataform_repository: 'dataform-repository' start_time: '2014-10-02T15:01:23Z' end_time: '2014-10-10T15:01:23Z' + key_name: 'my-key' test_env_vars: project_id: 'PROJECT_NAME' location: 'REGION' diff --git a/mmv1/products/datastream/Stream.yaml b/mmv1/products/datastream/Stream.yaml index 10c6ae72349e..fa4df7b8b3e0 100644 --- a/mmv1/products/datastream/Stream.yaml +++ b/mmv1/products/datastream/Stream.yaml @@ -216,6 +216,7 @@ examples: - name: 'datastream_stream_salesforce' primary_resource_id: 'default' vars: + stream_id: 'sf-stream' source_connection_profile_id: 'source-profile' destination_connection_profile_id: 'destination-profile' exclude_test: true diff --git a/mmv1/products/dialogflow/EntityType.yaml b/mmv1/products/dialogflow/EntityType.yaml index 179920721f21..c938ed0191b0 100644 --- a/mmv1/products/dialogflow/EntityType.yaml +++ b/mmv1/products/dialogflow/EntityType.yaml @@ -38,7 +38,7 @@ examples: - name: 'dialogflow_entity_type_basic' primary_resource_id: 'basic_entity_type' vars: - intent_name: 'basic-entity-type' + entity_type_name: 'basic-entity-type' exclude_test: true parameters: properties: diff --git a/mmv1/products/eventarc/Trigger.yaml b/mmv1/products/eventarc/Trigger.yaml index fcb4cd0a7a6c..35c1d03dd84e 100644 --- a/mmv1/products/eventarc/Trigger.yaml +++ b/mmv1/products/eventarc/Trigger.yaml @@ -48,6 +48,7 @@ examples: primary_resource_id: primary vars: trigger_name: some-trigger + network_attachment_name: network-attachment test_vars_overrides: 'network_attachment_name': 'acctest.BootstrapNetworkAttachment(t, "tf-test-eventarc-trigger-na", acctest.BootstrapSubnet(t, "tf-test-eventarc-trigger-subnet", acctest.BootstrapSharedTestNetwork(t, "tf-test-eventarc-trigger-network")))' test_env_vars: diff --git a/mmv1/products/firebaseapphosting/DefaultDomain.yaml b/mmv1/products/firebaseapphosting/DefaultDomain.yaml index 845823dcf53e..84e94a7a75da 100644 --- a/mmv1/products/firebaseapphosting/DefaultDomain.yaml +++ b/mmv1/products/firebaseapphosting/DefaultDomain.yaml @@ -33,6 +33,7 @@ examples: primary_resource_id: example vars: backend_id: 'dd-mini' + service_act_id: 'service-account' test_env_vars: project_id: 'PROJECT_NAME' test_vars_overrides: @@ -42,6 +43,7 @@ examples: primary_resource_id: example vars: backend_id: 'dd-full' + service_act_id: 'service-account' test_env_vars: project_id: 'PROJECT_NAME' test_vars_overrides: @@ -51,6 +53,7 @@ examples: primary_resource_id: example vars: backend_id: 'dd-disabled' + service_act_id: 'service-account' test_env_vars: project_id: 'PROJECT_NAME' test_vars_overrides: diff --git a/mmv1/products/firebaseapphosting/Domain.yaml b/mmv1/products/firebaseapphosting/Domain.yaml index d2907277f52c..f27993655a87 100644 --- a/mmv1/products/firebaseapphosting/Domain.yaml +++ b/mmv1/products/firebaseapphosting/Domain.yaml @@ -45,6 +45,8 @@ examples: primary_resource_id: example vars: backend_id: 'domain-mini' + service_act_id: 'sa-id' + domain_id: example.com test_env_vars: project_id: 'PROJECT_NAME' test_vars_overrides: @@ -55,6 +57,8 @@ examples: primary_resource_id: example vars: backend_id: 'domain-full' + service_act_id: 'sa-id' + domain_id: example.com test_env_vars: project_id: 'PROJECT_NAME' test_vars_overrides: diff --git a/mmv1/products/firebaseextensions/Instance.yaml b/mmv1/products/firebaseextensions/Instance.yaml index 71791e8e5c60..85d6a897f8b7 100644 --- a/mmv1/products/firebaseextensions/Instance.yaml +++ b/mmv1/products/firebaseextensions/Instance.yaml @@ -52,6 +52,7 @@ examples: instance-id: 'storage-resize-images' bucket_id: 'bucket-id' service-account-id: 's-a' + location: "us-central1" test_env_vars: project_id: 'PROJECT_NAME' test_vars_overrides: diff --git a/mmv1/products/netapp/Backup.yaml b/mmv1/products/netapp/Backup.yaml index 27bf9982c85a..091e3d8d245b 100644 --- a/mmv1/products/netapp/Backup.yaml +++ b/mmv1/products/netapp/Backup.yaml @@ -68,6 +68,7 @@ examples: volume_name: 'backup-volume' backup_vault_name: 'backup-vault' backup_name: 'test-backup' + network_name: 'network' test_vars_overrides: 'network_name': 'acctest.BootstrapSharedServiceNetworkingConnection(t, "gcnv-network-config-2", acctest.ServiceNetworkWithParentService("netapp.servicenetworking.goog"))' parameters: diff --git a/mmv1/products/securitycenter/OrganizationSccBigQueryExport.yaml b/mmv1/products/securitycenter/OrganizationSccBigQueryExport.yaml index 8a01eb3fa977..5a68d4bc8a59 100644 --- a/mmv1/products/securitycenter/OrganizationSccBigQueryExport.yaml +++ b/mmv1/products/securitycenter/OrganizationSccBigQueryExport.yaml @@ -45,7 +45,7 @@ examples: primary_resource_id: 'custom_big_query_export_config' vars: big_query_export_id: 'my-export' - dataset: 'my-dataset' + dataset_id: 'my-dataset' name: 'my-export' test_env_vars: org_id: 'ORG_ID' diff --git a/mmv1/products/securitycenterv2/OrganizationSccBigQueryExport.yaml b/mmv1/products/securitycenterv2/OrganizationSccBigQueryExport.yaml index 79e11daa14c3..8e415261d0ca 100644 --- a/mmv1/products/securitycenterv2/OrganizationSccBigQueryExport.yaml +++ b/mmv1/products/securitycenterv2/OrganizationSccBigQueryExport.yaml @@ -46,7 +46,7 @@ examples: primary_resource_id: 'custom_big_query_export_config' vars: big_query_export_id: 'my-export' - dataset: 'my-dataset' + dataset_id: 'my-dataset' name: 'my-export' test_env_vars: org_id: 'ORG_ID' diff --git a/mmv1/products/securitycenterv2/OrganizationSccBigQueryExports.yaml b/mmv1/products/securitycenterv2/OrganizationSccBigQueryExports.yaml index ccfcbb0f6752..2d3e382dec7b 100644 --- a/mmv1/products/securitycenterv2/OrganizationSccBigQueryExports.yaml +++ b/mmv1/products/securitycenterv2/OrganizationSccBigQueryExports.yaml @@ -47,7 +47,7 @@ examples: primary_resource_id: 'custom_big_query_export_config' vars: big_query_export_id: 'my-export' - dataset: 'my-dataset' + dataset_id: 'my-dataset' name: 'my-export' test_env_vars: org_id: 'ORG_ID' diff --git a/mmv1/products/vmwareengine/ExternalAddress.yaml b/mmv1/products/vmwareengine/ExternalAddress.yaml index ee3aa777699b..5e51a5143d24 100644 --- a/mmv1/products/vmwareengine/ExternalAddress.yaml +++ b/mmv1/products/vmwareengine/ExternalAddress.yaml @@ -62,6 +62,8 @@ examples: private_cloud_id: 'sample-pc' management_cluster_id: 'sample-mgmt-cluster' network_policy_id: 'sample-np' + test_env_vars: + region: 'REGION' # update tests will take care of all CRUD tests. Parent PC creation is expensive and node reservation is required. exclude_test: true parameters: diff --git a/mmv1/templates/terraform/examples/bigquery_analyticshub_listing_subscription_basic.tf.tmpl b/mmv1/templates/terraform/examples/bigquery_analyticshub_listing_subscription_basic.tf.tmpl index 752b377818ea..806abd0bb698 100644 --- a/mmv1/templates/terraform/examples/bigquery_analyticshub_listing_subscription_basic.tf.tmpl +++ b/mmv1/templates/terraform/examples/bigquery_analyticshub_listing_subscription_basic.tf.tmpl @@ -2,7 +2,7 @@ resource "google_bigquery_analytics_hub_data_exchange" "{{$.PrimaryResourceId}}" location = "US" data_exchange_id = "{{index $.Vars "data_exchange_id"}}" display_name = "{{index $.Vars "data_exchange_id"}}" - description = "{{index $.Vars "desc"}}" + description = "Test Description" } resource "google_bigquery_analytics_hub_listing" "{{$.PrimaryResourceId}}" { @@ -10,7 +10,7 @@ resource "google_bigquery_analytics_hub_listing" "{{$.PrimaryResourceId}}" { data_exchange_id = google_bigquery_analytics_hub_data_exchange.{{$.PrimaryResourceId}}.data_exchange_id listing_id = "{{index $.Vars "listing_id"}}" display_name = "{{index $.Vars "listing_id"}}" - description = "{{index $.Vars "desc"}}" + description = "Test Description" bigquery_dataset { dataset = google_bigquery_dataset.{{$.PrimaryResourceId}}.id @@ -20,7 +20,7 @@ resource "google_bigquery_analytics_hub_listing" "{{$.PrimaryResourceId}}" { resource "google_bigquery_dataset" "{{$.PrimaryResourceId}}" { dataset_id = "{{index $.Vars "listing_id"}}" friendly_name = "{{index $.Vars "listing_id"}}" - description = "{{index $.Vars "desc"}}" + description = "Test Description" location = "US" } diff --git a/mmv1/templates/terraform/examples/edgecontainer_local_control_plane_node_pool.tf.tmpl b/mmv1/templates/terraform/examples/edgecontainer_local_control_plane_node_pool.tf.tmpl index d7eaf014bf00..4edcdd696975 100644 --- a/mmv1/templates/terraform/examples/edgecontainer_local_control_plane_node_pool.tf.tmpl +++ b/mmv1/templates/terraform/examples/edgecontainer_local_control_plane_node_pool.tf.tmpl @@ -1,5 +1,5 @@ resource "google_edgecontainer_cluster" "{{$.PrimaryResourceId}}" { - name = "{{index $.Vars "edgecontainer_cluster_name"}}" + name = "default" location = "us-central1" authorization { diff --git a/mmv1/templates/terraform/examples/gkebackup_backupchannel_basic.tf.tmpl b/mmv1/templates/terraform/examples/gkebackup_backupchannel_basic.tf.tmpl index b31780a21afe..727a9d8de164 100644 --- a/mmv1/templates/terraform/examples/gkebackup_backupchannel_basic.tf.tmpl +++ b/mmv1/templates/terraform/examples/gkebackup_backupchannel_basic.tf.tmpl @@ -1,7 +1,7 @@ resource "google_gke_backup_backup_channel" "basic" { name = "{{index $.Vars "name"}}" location = "us-central1" - description = "{{index $.Vars "description"}}" + description = "Description" destination_project = "{{index $.Vars "destination_project"}}" labels = { "key": "some-value" } } diff --git a/mmv1/templates/terraform/examples/gkebackup_restorechannel_basic.tf.tmpl b/mmv1/templates/terraform/examples/gkebackup_restorechannel_basic.tf.tmpl index cfa7fcbff6d7..7baf60005bcc 100644 --- a/mmv1/templates/terraform/examples/gkebackup_restorechannel_basic.tf.tmpl +++ b/mmv1/templates/terraform/examples/gkebackup_restorechannel_basic.tf.tmpl @@ -1,7 +1,7 @@ resource "google_gke_backup_restore_channel" "basic" { name = "{{index $.Vars "name"}}" location = "us-central1" - description = "{{index $.Vars "description"}}" + description = "Description" destination_project = "{{index $.Vars "destination_project"}}" labels = { "key": "some-value" } } diff --git a/mmv1/templates/terraform/examples/instance_basic.tf.tmpl b/mmv1/templates/terraform/examples/instance_basic.tf.tmpl index ecbc1172ad0c..10f12c1b45a4 100644 --- a/mmv1/templates/terraform/examples/instance_basic.tf.tmpl +++ b/mmv1/templates/terraform/examples/instance_basic.tf.tmpl @@ -1,6 +1,6 @@ resource "google_compute_instance" "{{$.PrimaryResourceId}}" { name = "{{index $.Vars "instance_name"}}" - zone = "{{index $.Vars "zone_name"}}" + zone = "us-central1-a" machine_type = "e2-medium" boot_disk {