Add cai_asset_name_format to resource meta.yaml file (#15142)

Author: zli82016

mmv1/third_party/terraform/acctest/resource_inventory_reader.go

@@ -12,17 +12,20 @@ import (
 
 // ResourceYAMLMetadata represents the structure of the metadata files
 type ResourceYAMLMetadata struct {
-	Resource       string `yaml:"resource"`
-	ApiServiceName string `yaml:"api_service_name"`
-	SourceFile     string `yaml:"source_file"`
+	Resource           string `yaml:"resource"`
+	ApiServiceName     string `yaml:"api_service_name"`
+	CaiAssetNameFormat string `yaml:"cai_asset_name_format"`
+	SourceFile         string `yaml:"source_file"`
 }
 
 // Cache structures to avoid repeated file system operations
 var (
 	// Cache for API service names (resourceName -> apiServiceName)
-	apiServiceNameCache = make(map[string]string)
+	ApiServiceNameCache = NewGenericCache("unknown")
+	// Cache for CAI resource name format (resourceName -> CaiAssetNameFormat)
+	CaiAssetNameFormatCache = NewGenericCache("")
 	// Cache for service packages (resourceType -> servicePackage)
-	servicePackageCache = make(map[string]string)
+	ServicePackageCache = NewGenericCache("unknown")
 	// Flag to track if cache has been populated
 	cachePopulated = false
 	// Mutex to protect cache access
@@ -76,10 +79,14 @@ func PopulateMetadataCache() error {
 
 			// Store API service name in cache
 			if metadata.ApiServiceName != "" {
-				apiServiceNameCache[metadata.Resource] = metadata.ApiServiceName
+				ApiServiceNameCache.Set(metadata.Resource, metadata.ApiServiceName)
 				apiNameCount++
 			}
 
+			if metadata.CaiAssetNameFormat != "" {
+				CaiAssetNameFormatCache.Set(metadata.Resource, metadata.CaiAssetNameFormat)
+			}
+
 			// Extract and store service package in cache
 			pathParts := strings.Split(path, string(os.PathSeparator))
 			servicesIndex := -1
@@ -92,7 +99,7 @@ func PopulateMetadataCache() error {
 
 			if servicesIndex >= 0 && len(pathParts) > servicesIndex+1 {
 				servicePackage := pathParts[servicesIndex+1] // The part after "services"
-				servicePackageCache[metadata.Resource] = servicePackage
+				ServicePackageCache.Set(metadata.Resource, servicePackage)
 				servicePkgCount++
 			}
 		}
@@ -109,48 +116,42 @@ func PopulateMetadataCache() error {
 	return nil
 }
 
-// GetAPIServiceNameForResource finds the api_service_name for a given resource name
-// If projectRoot is empty, it will attempt to find the project root automatically
-func GetAPIServiceNameForResource(resourceName string) string {
-	// Make sure cache is populated
-	if !cachePopulated {
-		if err := PopulateMetadataCache(); err != nil {
-			return "failed_to_populate_metadata_cache"
-		}
-	}
-
-	// Check cache
-	cacheMutex.RLock()
-	apiServiceName, found := apiServiceNameCache[resourceName]
-	cacheMutex.RUnlock()
+type GenericCache struct {
+	mu           sync.RWMutex
+	data         map[string]string
+	defaultValue string
+}
 
-	if !found {
-		return "unknown"
+// NewGenericCache initializes a new GenericCache with a default value.
+func NewGenericCache(defaultValue string) *GenericCache {
+	return &GenericCache{
+		data:         make(map[string]string),
+		defaultValue: defaultValue,
 	}
-
-	return apiServiceName
 }
 
-// GetServicePackageForResourceType finds the service package for a given resource type
-// If projectRoot is empty, it will attempt to find the project root automatically
-func GetServicePackageForResourceType(resourceType string) string {
+// Get retrieves a value from the cache, returning the default if not found.
+func (c *GenericCache) Get(key string) string {
 	// Make sure cache is populated
 	if !cachePopulated {
 		if err := PopulateMetadataCache(); err != nil {
 			return "failed_to_populate_metadata_cache"
 		}
 	}
 
-	// Check cache
-	cacheMutex.RLock()
-	servicePackage, found := servicePackageCache[resourceType]
-	cacheMutex.RUnlock()
-
-	if !found {
-		return "unknown"
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	value, ok := c.data[key]
+	if !ok {
+		return c.defaultValue
 	}
+	return value
+}
 
-	return servicePackage
+func (c *GenericCache) Set(key, value string) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.data[key] = value
 }
 
 // getServicesDir returns the path to the services directory

mmv1/third_party/terraform/acctest/resource_inventory_test.go

@@ -34,14 +34,14 @@ func TestResourceInventoryMetadataFound(t *testing.T) {
 		// t.Logf("Checking metadata for resource: %s", resourceType)
 
 		// Check for service package
-		servicePackage := acctest.GetServicePackageForResourceType(resourceType)
+		servicePackage := acctest.ServicePackageCache.Get(resourceType)
 		if servicePackage == "unknown" {
 			// t.Logf("WARNING: Could not find service package for resource %s: %v", resourceType)
 			missingServicePkg++
 			missingServicePkgResources[resourceType] = true
 		}
 
-		apiServiceName := acctest.GetAPIServiceNameForResource(resourceType)
+		apiServiceName := acctest.ApiServiceNameCache.Get(resourceType)
 		// Check for API service name
 		if apiServiceName == "unknown" {
 			// t.Logf("WARNING: Could not find API service name for resource %s: %v", resourceType)

mmv1/third_party/terraform/acctest/tgc_utils.go

@@ -95,26 +95,43 @@ func CollectAllTgcMetadata(tgcPayload TgcMetadataPayload) resource.TestCheckFunc
 			}
 
 			// Resolve the CAI asset name
-			apiServiceName := GetAPIServiceNameForResource(metadata.ResourceType)
-			if apiServiceName == "unknown" || apiServiceName == "failed_to_populate_metadata_cache" {
-				log.Printf("[DEBUG]TGC Terraform error: unknown resource type %s", metadata.ResourceType)
-				metadata.CaiAssetNames = []string{apiServiceName}
-			} else {
-				var rName string
-				switch metadata.ResourceType {
-				case "google_project":
-					rName = fmt.Sprintf("projects/%s", rState.Primary.Attributes["number"])
-				default:
-					rName = rState.Primary.ID
-				}
+			caiAssetNameFormat := CaiAssetNameFormatCache.Get(metadata.ResourceType)
+			if caiAssetNameFormat == "" || caiAssetNameFormat == "failed_to_populate_metadata_cache" {
+				log.Printf("[DEBUG]TGC Terraform error: unknown CAI asset name format for resource type %s", caiAssetNameFormat)
+
+				apiServiceName := ApiServiceNameCache.Get(metadata.ResourceType)
+				if apiServiceName == "unknown" || apiServiceName == "failed_to_populate_metadata_cache" {
+					log.Printf("[DEBUG]TGC Terraform error: unknown resource type %s", metadata.ResourceType)
+					metadata.CaiAssetNames = []string{apiServiceName}
+				} else {
+					var rName string
+					switch metadata.ResourceType {
+					case "google_project":
+						rName = fmt.Sprintf("projects/%s", rState.Primary.Attributes["number"])
+					default:
+						rName = rState.Primary.ID
+					}
+
+					if _, ok := serviceWithProjectNumber[metadata.Service]; ok {
+						rName = strings.Replace(rName, projectId, projectNumber, 1)
+					}
 
-				if _, ok := serviceWithProjectNumber[metadata.Service]; ok {
-					rName = strings.Replace(rName, projectId, projectNumber, 1)
+					metadata.CaiAssetNames = []string{fmt.Sprintf("//%s/%s", apiServiceName, rName)}
+				}
+			} else {
+				paramsMap := make(map[string]any, 0)
+				params := extractIdentifiers(caiAssetNameFormat)
+				for _, param := range params {
+					v := rState.Primary.Attributes[param]
+					paramsMap[param] = v
 				}
 
-				metadata.CaiAssetNames = []string{fmt.Sprintf("//%s/%s", apiServiceName, rName)}
+				caiAssetName := replacePlaceholders(caiAssetNameFormat, paramsMap)
+				metadata.CaiAssetNames = []string{caiAssetName}
 			}
 
+			log.Printf("[DEBUG] CaiAssetNames %#v", metadata.CaiAssetNames)
+
 			// Resolve auto IDs in import metadata
 			if metadata.ImportMetadata.Id != "" {
 				metadata.ImportMetadata.Id = strings.Replace(metadata.ImportMetadata.Id, "<AUTO_ID>", rState.Primary.ID, 1)
@@ -138,6 +155,35 @@ func CollectAllTgcMetadata(tgcPayload TgcMetadataPayload) resource.TestCheckFunc
 	}
 }
 
+// For example, for the url "projects/{{project}}/schemas/{{schema}}",
+// the identifiers are "project", "schema".
+func extractIdentifiers(url string) []string {
+	matches := regexp.MustCompile(`\{\{%?(\w+)\}\}`).FindAllStringSubmatch(url, -1)
+	var result []string
+	for _, match := range matches {
+		result = append(result, match[1])
+	}
+	return result
+}
+
+// It replaces all instances of {{key}} in the template with the
+// corresponding value from the parameters map.
+func replacePlaceholders(template string, params map[string]any) string {
+	re := regexp.MustCompile(`\{\{([a-zA-Z0-9_]+)\}\}`)
+
+	result := re.ReplaceAllStringFunc(template, func(match string) string {
+		key := strings.Trim(match, "{}")
+
+		if value, ok := params[key]; ok {
+			return value.(string)
+		}
+
+		return match
+	})
+
+	return result
+}
+
 // parseResources extracts all resources from a Terraform configuration string
 func parseResources(config string) []string {
 	// This regex matches resource blocks in Terraform configurations
@@ -246,7 +292,7 @@ func extendWithTGCData(t *testing.T, c resource.TestCase) resource.TestCase {
 						ResourceType:    resourceType,
 						ResourceAddress: res,
 						ImportMetadata:  importMeta,
-						Service:         GetServicePackageForResourceType(resourceType),
+						Service:         ApiServiceNameCache.Get(resourceType),
 						// CaiAssetNames will be populated at runtime in the check function
 					}
 				}

mmv1/third_party/terraform/services/appengine/resource_app_engine_application_meta.yaml.tmpl

@@ -7,6 +7,7 @@ api_version: 'v1beta'
 api_version: 'v1'
 {{- end }}
 api_resource_type_kind: 'Application'
+cai_asset_name_format: '//appengine.googleapis.com/{{"{{"}}name{{"}}"}}'
 fields:
   - field: 'app_id'
   - field: 'auth_domain'

mmv1/third_party/terraform/services/bigtable/resource_bigtable_instance_meta.yaml

@@ -1,6 +1,7 @@
 resource: 'google_bigtable_instance'
 generation_type: 'handwritten'
 api_service_name: 'bigtableadmin.googleapis.com'
+cai_asset_name_format: '//bigtable.googleapis.com/projects/{{project}}/instances/{{name}}'
 api_version: 'v2'
 api_resource_type_kind: 'Instance'
 fields:

mmv1/third_party/terraform/services/dataproc/resource_dataproc_workflow_template_meta.yaml.tmpl

@@ -5,6 +5,7 @@ api_version: 'v1'
 api_resource_type_kind: 'WorkflowTemplate'
 api_variant_patterns:
   - 'projects/{project}/locations/{location}/workflowTemplates/{workflowTemplate}'
+cai_asset_name_format: {{"//dataproc.googleapis.com/projects/{{project}}/regions/{{location}}/workflowTemplates/{{name}}"}}
 fields:
   - field: 'create_time'
   - field: 'dag_timeout'

mmv1/third_party/terraform/services/tags/resource_tags_location_tag_binding_meta.yaml

@@ -3,6 +3,7 @@ generation_type: 'handwritten'
 api_service_name: 'cloudresourcemanager.googleapis.com'
 api_version: 'v3'
 api_resource_type_kind: 'TagBinding'
+cai_asset_name_format: '//cloudresourcemanager.googleapis.com/{{name}}'
 fields:
   - field: 'location'
   - field: 'name'