tgc-revival: parse values from HCL

mmv1/third_party/tgc_next/test/assert_test_files.go

@@ -185,7 +185,11 @@ func testSingleResource(t *testing.T, testName string, testData ResourceTestData
 		return fmt.Errorf("missing hcl after cai2hcl conversion for resource %s", testData.ResourceType)
 	}
 
-	ignoredFieldSet := make(map[string]struct{}, 0)
+	if os.Getenv("WRITE_FILES") != "" {
+		writeJSONFile(fmt.Sprintf("%s_export_attrs", testName), exportResources)
+	}
+
+	ignoredFieldSet := make(map[string]any, 0)
 	for _, f := range ignoredFields {
 		ignoredFieldSet[f] = struct{}{}
 	}
@@ -323,9 +327,9 @@ func getAncestryCache(assets []caiasset.Asset) (map[string]string, string) {
 }
 
 // Compares HCL and finds all of the keys in map1 that are not in map2
-func compareHCLFields(map1, map2, ignoredFields map[string]struct{}) []string {
+func compareHCLFields(map1, map2, ignoredFields map[string]any) []string {
 	var missingKeys []string
-	for key := range map1 {
+	for key, _ := range map1 {
 		if isIgnored(key, ignoredFields) {
 			continue
 		}
@@ -339,7 +343,7 @@ func compareHCLFields(map1, map2, ignoredFields map[string]struct{}) []string {
 }
 
 // Returns true if the given key should be ignored according to the given set of ignored fields.
-func isIgnored(key string, ignoredFields map[string]struct{}) bool {
+func isIgnored(key string, ignoredFields map[string]any) bool {
 	// Check for exact match first.
 	if _, ignored := ignoredFields[key]; ignored {
 		return true

mmv1/third_party/tgc_next/test/hcl.go

@@ -9,9 +9,11 @@ import (
 	"github.com/hashicorp/hcl/v2"
 	"github.com/hashicorp/hcl/v2/hclparse"
 	"github.com/hashicorp/hcl/v2/hclsyntax"
+	"github.com/zclconf/go-cty/cty"
+	"github.com/zclconf/go-cty/cty/gocty"
 )
 
-func parseHCLBytes(src []byte, filePath string) (map[string]map[string]struct{}, error) {
+func parseHCLBytes(src []byte, filePath string) (map[string]map[string]any, error) {
 	parser := hclparse.NewParser()
 	hclFile, diags := parser.ParseHCL(src, filePath)
 	if diags.HasErrors() {
@@ -22,7 +24,7 @@ func parseHCLBytes(src []byte, filePath string) (map[string]map[string]struct{},
 		return nil, fmt.Errorf("parsed HCL file %s is nil cannot proceed", filePath)
 	}
 
-	parsed := make(map[string]map[string]struct{})
+	parsed := make(map[string]map[string]any)
 
 	for _, block := range hclFile.Body.(*hclsyntax.Body).Blocks {
 		if block.Type == "resource" {
@@ -43,7 +45,7 @@ func parseHCLBytes(src []byte, filePath string) (map[string]map[string]struct{},
 				}
 			}
 
-			flattenedAttrs := make(map[string]struct{})
+			flattenedAttrs := make(map[string]any)
 			flatten(attrs, "", flattenedAttrs)
 			parsed[addr] = flattenedAttrs
 		}
@@ -61,7 +63,7 @@ func parseHCLBody(body hcl.Body) (
 
 	if syntaxBody, ok := body.(*hclsyntax.Body); ok {
 		for _, attr := range syntaxBody.Attributes {
-			insert(struct{}{}, attr.Name, attributes)
+			insert(getValue(attr.Expr), attr.Name, attributes)
 		}
 
 		for _, block := range syntaxBody.Blocks {
@@ -96,39 +98,43 @@ func insert(data any, key string, parent map[string]any) {
 	}
 }
 
-func flatten(data any, prefix string, result map[string]struct{}) {
+func flatten(data any, prefix string, result map[string]any) {
 	switch v := data.(type) {
 	case map[string]any:
-		for key, value := range v {
-			newPrefix := key
-			if prefix != "" {
-				newPrefix = prefix + "." + key
+		if len(v) == 0 && prefix != "" {
+			result[prefix] = v
+		} else {
+			for key, value := range v {
+				newPrefix := key
+				if prefix != "" {
+					newPrefix = prefix + "." + key
+				}
+				flatten(value, newPrefix, result)
 			}
-			flatten(value, newPrefix, result)
 		}
 	case []any:
 		flattenSlice(prefix, v, result)
 	default:
 		if prefix != "" {
-			result[prefix] = struct{}{}
+			result[prefix] = v
 		}
 	}
 }
 
-func flattenSlice(prefix string, v []any, result map[string]struct{}) {
+func flattenSlice(prefix string, v []any, result map[string]any) {
 	if len(v) == 0 && prefix != "" {
 		result[prefix] = struct{}{}
 		return
 	}
 
 	type sortableElement struct {
 		flatKeys  string
-		flattened map[string]struct{}
+		flattened map[string]any
 	}
 
 	sortable := make([]sortableElement, len(v))
 	for i, value := range v {
-		flattened := make(map[string]struct{})
+		flattened := make(map[string]any)
 		flatten(value, "", flattened)
 		keys := make([]string, 0, len(flattened))
 		for k := range flattened {
@@ -152,13 +158,88 @@ func flattenSlice(prefix string, v []any, result map[string]struct{}) {
 				result[newPrefix] = struct{}{}
 			}
 		} else {
-			for k := range element.flattened {
+			for k, v := range element.flattened {
 				newKey := newPrefix
 				if k != "" {
 					newKey = newPrefix + "." + k
 				}
-				result[newKey] = struct{}{}
+				result[newKey] = v
 			}
 		}
 	}
 }
+
+// Gets the value of the expression of an attribute
+func getValue(expr hcl.Expression) any {
+	switch expr := expr.(type) {
+	case *hclsyntax.ScopeTraversalExpr:
+		// Example: id = google_instance.web.id
+		return getTraveralExprVal(expr.Traversal)
+	case *hclsyntax.LiteralValueExpr:
+		// Example: region = "us-west1"
+		return convertValue(expr.Val)
+	case *hclsyntax.TemplateExpr:
+		// Example: ip_address = "IP: ${var.ip}"
+		vStr := ""
+		parts := expr.Parts
+		for _, part := range parts {
+			vStr += getValue(part).(string)
+		}
+		return vStr
+	case *hclsyntax.TupleConsExpr:
+		// Example: methods = ["GET", "POST", "DELETE"]
+		exprV := make([]string, 0, len(expr.Exprs))
+		for _, elem := range expr.Exprs {
+			exprV = append(exprV, fmt.Sprint(getValue(elem)))
+		}
+		return strings.Join(exprV, ",")
+	case *hclsyntax.ObjectConsKeyExpr:
+		// Example: labels = {(local.key_name) = "value"}
+		return getValue(expr.Wrapped).(string)
+	case *hclsyntax.ObjectConsExpr:
+		// Example: tags = { Env = "dev", Owner = var.user }
+		return map[string]any{}
+	case *hclsyntax.ParenthesesExpr:
+		// Example: (local.key_name)
+		return getValue(expr.Expression)
+	default:
+		log.Printf("Unsupported expression type: %T", expr)
+		return nil
+	}
+}
+
+// Converts a literal cty.Value to a standard Go value
+func convertValue(val cty.Value) any {
+	switch val.Type() {
+	case cty.Number:
+		f, _ := val.AsBigFloat().Float64()
+		return f
+	case cty.String:
+		return val.AsString()
+	case cty.Bool:
+		var v bool
+		_ = gocty.FromCtyValue(val, &v)
+		return v
+	default:
+		return ""
+	}
+}
+
+// Gets the value for the traveral expression (e.g. "google_pubsub_topic.example.id")
+func getTraveralExprVal(traversal hcl.Traversal) string {
+	exprV := make([]string, 0, len(traversal))
+
+	for _, v := range traversal {
+		switch v := v.(type) {
+		// The starting point of the traversal (e.g. "google_pubsub_topic)
+		case hcl.TraverseRoot:
+			exprV = append(exprV, v.Name)
+		// A list of hclsyntax.Traversal objects, which represent each step in the path.
+		// In "google_pubsub_topic.example.id", the steps are "example", "id"
+		case hcl.TraverseAttr:
+			exprV = append(exprV, v.Name)
+		}
+	}
+
+	return strings.Join(exprV, ".")
+}

mmv1/third_party/tgc_next/test/hcl_test.go

@@ -76,6 +76,19 @@ resource "google_compute_firewall" "default" {
 
   source_tags = ["web"]
 }
+`
+	mapHCL = `
+resource "google_bigquery_dataset" "dataset" {
+  dataset_id    = "datasetjlgukul2im"
+  description   = "This is a test description"
+  friendly_name = "test"
+  location      = "EU"
+  project       = "ci-test-project-nightly-beta"
+  resource_tags = {
+    "ci-test-project-nightly-beta/tf_test_tag_key1jlgukul2im" = "tf_test_tag_value1jlgukul2im"
+    "ci-test-project-nightly-beta/tf_test_tag_key2jlgukul2im" = "tf_test_tag_value2jlgukul2im"
+  }
+}
 `
 )
 
@@ -84,70 +97,84 @@ func TestParseHCLBytes(t *testing.T) {
 	cases := []struct {
 		name      string
 		hcl       string
-		exp       map[string]map[string]struct{}
+		exp       map[string]map[string]any
 		expectErr bool
 	}{
 		{
 			name: "basic",
 			hcl:  basicHCL,
-			exp: map[string]map[string]struct{}{
+			exp: map[string]map[string]any{
 				"google_project_service.project": {
-					"service": {},
+					"service": "iam.googleapis.com",
 				},
 			},
 		},
 		{
 			name: "nested blocks",
 			hcl:  nestedBlocksHCL,
-			exp: map[string]map[string]struct{}{
+			exp: map[string]map[string]any{
 				"google_storage_bucket.bucket": {
-					"name":                         {},
-					"location":                     {},
-					"force_destroy":                {},
-					"lifecycle_rule.action.type":   {},
-					"lifecycle_rule.condition.age": {},
+					"name":                         "my-bucket",
+					"location":                     "US",
+					"force_destroy":                true,
+					"lifecycle_rule.action.type":   "Delete",
+					"lifecycle_rule.condition.age": float64(30),
 				},
 			},
 		},
 		{
 			name: "multiple resources",
 			hcl:  multipleResourcesHCL,
-			exp: map[string]map[string]struct{}{
+			exp: map[string]map[string]any{
 				"google_project_service.project": {
-					"service": {},
+					"service": "iam.googleapis.com",
 				},
 				"google_storage_bucket.bucket": {
-					"name": {},
+					"name": "my-bucket",
 				},
 			},
 		},
 		{
 			name: "resource with a list of nested objects",
 			hcl:  listOfNestedObjectsHCL,
-			exp: map[string]map[string]struct{}{
+			exp: map[string]map[string]any{
 				"google_compute_firewall.default": {
-					"allow.0.ports":    {}, // "ports" appears in first element due to sorting
-					"allow.0.protocol": {},
-					"allow.1.protocol": {},
-					"name":             {},
-					"network":          {},
-					"source_tags":      {},
+					"allow.0.ports":    "80,8080,1000-2000", // "ports" appears in first element due to sorting
+					"allow.0.protocol": "tcp",
+					"allow.1.protocol": "icmp",
+					"name":             "test-firewall",
+					"network":          "google_compute_network.default.name",
+					"source_tags":      "web",
 				},
 			},
 		},
 		{
 			name: "resource with a list of multi-level nested objects",
 			hcl:  listOfMultiLevelNestedObjectsHCL,
-			exp: map[string]map[string]struct{}{
+			exp: map[string]map[string]any{
 				"google_compute_firewall.default": {
-					"allow.0.a_second_level.0.a": {},
-					"allow.0.a_second_level.1.b": {},
-					"allow.0.protocol":           {},
-					"allow.1.ports":              {},
-					"allow.1.protocol":           {},
-					"name":                       {},
-					"network":                    {},
-					"source_tags":                {},
+					"allow.0.a_second_level.0.a": false,
+					"allow.0.a_second_level.1.b": true,
+					"allow.0.protocol":           "icmp",
+					"allow.1.ports":              "80,8080,1000-2000",
+					"allow.1.protocol":           "tcp",
+					"name":                       "test-firewall",
+					"network":                    "google_compute_network.default.name",
+					"source_tags":                "web",
+				},
+			},
+		},
+		{
+			name: "resource with map",
+			hcl:  mapHCL,
+			exp: map[string]map[string]any{
+				"google_bigquery_dataset.dataset": {
+					"dataset_id":    "datasetjlgukul2im",
+					"description":   "This is a test description",
+					"friendly_name": "test",
+					"location":      "EU",
+					"project":       "ci-test-project-nightly-beta",
+					"resource_tags": map[string]any{},
 				},
 			},
 		},

mmv1/third_party/tgc_next/test/setup.go

@@ -42,7 +42,7 @@ type TgcMetadataPayload struct {
 }
 
 type ResourceTestData struct {
-	ParsedRawConfig  map[string]struct{} `json:"parsed_raw_config"`
+	ParsedRawConfig  map[string]any `json:"parsed_raw_config"`
 	ResourceMetadata `json:"resource_metadata"`
 }
 
@@ -53,9 +53,9 @@ type StepTestData struct {
 }
 
 type Resource struct {
-	Type       string              `json:"type"`
-	Name       string              `json:"name"`
-	Attributes map[string]struct{} `json:"attributes"`
+	Type       string         `json:"type"`
+	Name       string         `json:"name"`
+	Attributes map[string]any `json:"attributes"`
 }
 
 const (
@@ -205,7 +205,11 @@ func prepareTestData(testName string, stepNumber int, retries int) (*StepTestDat
 		}
 
 		if len(rawResourceConfigs) == 0 {
-			return nil, fmt.Errorf("Test %s fails: raw config is unavailable", testName)
+			return nil, fmt.Errorf("test %s fails: raw config is unavailable", testName)
+		}
+
+		if os.Getenv("WRITE_FILES") != "" {
+			writeJSONFile(fmt.Sprintf("%s_attrs", testName), rawResourceConfigs)
 		}
 
 		rawConfigMap := convertToConfigMap(rawResourceConfigs)
@@ -255,8 +259,8 @@ func parseResourceConfigs(filePath string) ([]Resource, error) {
 }
 
 // Converts the slice to map with resource address as the key
-func convertToConfigMap(resources []Resource) map[string]map[string]struct{} {
-	configMap := make(map[string]map[string]struct{}, 0)
+func convertToConfigMap(resources []Resource) map[string]map[string]any {
+	configMap := make(map[string]map[string]any, 0)
 
 	for _, r := range resources {
 		addr := fmt.Sprintf("%s.%s", r.Type, r.Name)