Skip to content

Commit 7a70e4d

Browse files
zli82016zli82016
authored
tgc-revival: parse values from HCL (#15208)
1 parent 7db0926 commit 7a70e4d

File tree

4 files changed

+171
-55
lines changed

4 files changed

+171
-55
lines changed

‎mmv1/third_party/tgc_next/test/assert_test_files.go‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -185,7 +185,11 @@ func testSingleResource(t *testing.T, testName string, testData ResourceTestData
185185
return fmt.Errorf("missing hcl after cai2hcl conversion for resource %s", testData.ResourceType)
186186
}
187187

188-
ignoredFieldSet := make(map[string]struct{}, 0)
188+
if os.Getenv("WRITE_FILES") != "" {
189+
writeJSONFile(fmt.Sprintf("%s_export_attrs", testName), exportResources)
190+
}
191+
192+
ignoredFieldSet := make(map[string]any, 0)
189193
for _, f := range ignoredFields {
190194
ignoredFieldSet[f] = struct{}{}
191195
}
@@ -323,9 +327,9 @@ func getAncestryCache(assets []caiasset.Asset) (map[string]string, string) {
323327
}
324328

325329
// Compares HCL and finds all of the keys in map1 that are not in map2
326-
func compareHCLFields(map1, map2, ignoredFields map[string]struct{}) []string {
330+
func compareHCLFields(map1, map2, ignoredFields map[string]any) []string {
327331
var missingKeys []string
328-
for key := range map1 {
332+
for key, _ := range map1 {
329333
if isIgnored(key, ignoredFields) {
330334
continue
331335
}
@@ -339,7 +343,7 @@ func compareHCLFields(map1, map2, ignoredFields map[string]struct{}) []string {
339343
}
340344

341345
// Returns true if the given key should be ignored according to the given set of ignored fields.
342-
func isIgnored(key string, ignoredFields map[string]struct{}) bool {
346+
func isIgnored(key string, ignoredFields map[string]any) bool {
343347
// Check for exact match first.
344348
if _, ignored := ignoredFields[key]; ignored {
345349
return true

‎mmv1/third_party/tgc_next/test/hcl.go‎

Lines changed: 97 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,11 @@ import (
99
"github.com/hashicorp/hcl/v2"
1010
"github.com/hashicorp/hcl/v2/hclparse"
1111
"github.com/hashicorp/hcl/v2/hclsyntax"
12+
"github.com/zclconf/go-cty/cty"
13+
"github.com/zclconf/go-cty/cty/gocty"
1214
)
1315

14-
func parseHCLBytes(src []byte, filePath string) (map[string]map[string]struct{}, error) {
16+
func parseHCLBytes(src []byte, filePath string) (map[string]map[string]any, error) {
1517
parser := hclparse.NewParser()
1618
hclFile, diags := parser.ParseHCL(src, filePath)
1719
if diags.HasErrors() {
@@ -22,7 +24,7 @@ func parseHCLBytes(src []byte, filePath string) (map[string]map[string]struct{},
2224
return nil, fmt.Errorf("parsed HCL file %s is nil cannot proceed", filePath)
2325
}
2426

25-
parsed := make(map[string]map[string]struct{})
27+
parsed := make(map[string]map[string]any)
2628

2729
for _, block := range hclFile.Body.(*hclsyntax.Body).Blocks {
2830
if block.Type == "resource" {
@@ -43,7 +45,7 @@ func parseHCLBytes(src []byte, filePath string) (map[string]map[string]struct{},
4345
}
4446
}
4547

46-
flattenedAttrs := make(map[string]struct{})
48+
flattenedAttrs := make(map[string]any)
4749
flatten(attrs, "", flattenedAttrs)
4850
parsed[addr] = flattenedAttrs
4951
}
@@ -61,7 +63,7 @@ func parseHCLBody(body hcl.Body) (
6163

6264
if syntaxBody, ok := body.(*hclsyntax.Body); ok {
6365
for _, attr := range syntaxBody.Attributes {
64-
insert(struct{}{}, attr.Name, attributes)
66+
insert(getValue(attr.Expr), attr.Name, attributes)
6567
}
6668

6769
for _, block := range syntaxBody.Blocks {
@@ -96,39 +98,43 @@ func insert(data any, key string, parent map[string]any) {
9698
}
9799
}
98100

99-
func flatten(data any, prefix string, result map[string]struct{}) {
101+
func flatten(data any, prefix string, result map[string]any) {
100102
switch v := data.(type) {
101103
case map[string]any:
102-
for key, value := range v {
103-
newPrefix := key
104-
if prefix != "" {
105-
newPrefix = prefix + "." + key
104+
if len(v) == 0 && prefix != "" {
105+
result[prefix] = v
106+
} else {
107+
for key, value := range v {
108+
newPrefix := key
109+
if prefix != "" {
110+
newPrefix = prefix + "." + key
111+
}
112+
flatten(value, newPrefix, result)
106113
}
107-
flatten(value, newPrefix, result)
108114
}
109115
case []any:
110116
flattenSlice(prefix, v, result)
111117
default:
112118
if prefix != "" {
113-
result[prefix] = struct{}{}
119+
result[prefix] = v
114120
}
115121
}
116122
}
117123

118-
func flattenSlice(prefix string, v []any, result map[string]struct{}) {
124+
func flattenSlice(prefix string, v []any, result map[string]any) {
119125
if len(v) == 0 && prefix != "" {
120126
result[prefix] = struct{}{}
121127
return
122128
}
123129

124130
type sortableElement struct {
125131
flatKeys string
126-
flattened map[string]struct{}
132+
flattened map[string]any
127133
}
128134

129135
sortable := make([]sortableElement, len(v))
130136
for i, value := range v {
131-
flattened := make(map[string]struct{})
137+
flattened := make(map[string]any)
132138
flatten(value, "", flattened)
133139
keys := make([]string, 0, len(flattened))
134140
for k := range flattened {
@@ -152,13 +158,88 @@ func flattenSlice(prefix string, v []any, result map[string]struct{}) {
152158
result[newPrefix] = struct{}{}
153159
}
154160
} else {
155-
for k := range element.flattened {
161+
for k, v := range element.flattened {
156162
newKey := newPrefix
157163
if k != "" {
158164
newKey = newPrefix + "." + k
159165
}
160-
result[newKey] = struct{}{}
166+
result[newKey] = v
161167
}
162168
}
163169
}
164170
}
171+
172+
// Gets the value of the expression of an attribute
173+
func getValue(expr hcl.Expression) any {
174+
switch expr := expr.(type) {
175+
case *hclsyntax.ScopeTraversalExpr:
176+
// Example: id = google_instance.web.id
177+
return getTraveralExprVal(expr.Traversal)
178+
case *hclsyntax.LiteralValueExpr:
179+
// Example: region = "us-west1"
180+
return convertValue(expr.Val)
181+
case *hclsyntax.TemplateExpr:
182+
// Example: ip_address = "IP: ${var.ip}"
183+
vStr := ""
184+
parts := expr.Parts
185+
for _, part := range parts {
186+
vStr += getValue(part).(string)
187+
}
188+
return vStr
189+
case *hclsyntax.TupleConsExpr:
190+
// Example: methods = ["GET", "POST", "DELETE"]
191+
exprV := make([]string, 0, len(expr.Exprs))
192+
for _, elem := range expr.Exprs {
193+
exprV = append(exprV, fmt.Sprint(getValue(elem)))
194+
}
195+
return strings.Join(exprV, ",")
196+
case *hclsyntax.ObjectConsKeyExpr:
197+
// Example: labels = {(local.key_name) = "value"}
198+
return getValue(expr.Wrapped).(string)
199+
case *hclsyntax.ObjectConsExpr:
200+
// Example: tags = { Env = "dev", Owner = var.user }
201+
return map[string]any{}
202+
case *hclsyntax.ParenthesesExpr:
203+
// Example: (local.key_name)
204+
return getValue(expr.Expression)
205+
default:
206+
log.Printf("Unsupported expression type: %T", expr)
207+
return nil
208+
}
209+
}
210+
211+
// Converts a literal cty.Value to a standard Go value
212+
func convertValue(val cty.Value) any {
213+
switch val.Type() {
214+
case cty.Number:
215+
f, _ := val.AsBigFloat().Float64()
216+
return f
217+
case cty.String:
218+
return val.AsString()
219+
case cty.Bool:
220+
var v bool
221+
_ = gocty.FromCtyValue(val, &v)
222+
return v
223+
default:
224+
return ""
225+
}
226+
}
227+
228+
// Gets the value for the traveral expression (e.g. "google_pubsub_topic.example.id")
229+
func getTraveralExprVal(traversal hcl.Traversal) string {
230+
exprV := make([]string, 0, len(traversal))
231+
232+
for _, v := range traversal {
233+
switch v := v.(type) {
234+
// The starting point of the traversal (e.g. "google_pubsub_topic)
235+
case hcl.TraverseRoot:
236+
exprV = append(exprV, v.Name)
237+
// A list of hclsyntax.Traversal objects, which represent each step in the path.
238+
// In "google_pubsub_topic.example.id", the steps are "example", "id"
239+
case hcl.TraverseAttr:
240+
exprV = append(exprV, v.Name)
241+
}
242+
}
243+
244+
return strings.Join(exprV, ".")
245+
}

‎mmv1/third_party/tgc_next/test/hcl_test.go‎

Lines changed: 55 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,19 @@ resource "google_compute_firewall" "default" {
7676
7777
source_tags = ["web"]
7878
}
79+
`
80+
mapHCL = `
81+
resource "google_bigquery_dataset" "dataset" {
82+
dataset_id = "datasetjlgukul2im"
83+
description = "This is a test description"
84+
friendly_name = "test"
85+
location = "EU"
86+
project = "ci-test-project-nightly-beta"
87+
resource_tags = {
88+
"ci-test-project-nightly-beta/tf_test_tag_key1jlgukul2im" = "tf_test_tag_value1jlgukul2im"
89+
"ci-test-project-nightly-beta/tf_test_tag_key2jlgukul2im" = "tf_test_tag_value2jlgukul2im"
90+
}
91+
}
7992
`
8093
)
8194

@@ -84,70 +97,84 @@ func TestParseHCLBytes(t *testing.T) {
8497
cases := []struct {
8598
name string
8699
hcl string
87-
exp map[string]map[string]struct{}
100+
exp map[string]map[string]any
88101
expectErr bool
89102
}{
90103
{
91104
name: "basic",
92105
hcl: basicHCL,
93-
exp: map[string]map[string]struct{}{
106+
exp: map[string]map[string]any{
94107
"google_project_service.project": {
95-
"service": {},
108+
"service": "iam.googleapis.com",
96109
},
97110
},
98111
},
99112
{
100113
name: "nested blocks",
101114
hcl: nestedBlocksHCL,
102-
exp: map[string]map[string]struct{}{
115+
exp: map[string]map[string]any{
103116
"google_storage_bucket.bucket": {
104-
"name": {},
105-
"location": {},
106-
"force_destroy": {},
107-
"lifecycle_rule.action.type": {},
108-
"lifecycle_rule.condition.age": {},
117+
"name": "my-bucket",
118+
"location": "US",
119+
"force_destroy": true,
120+
"lifecycle_rule.action.type": "Delete",
121+
"lifecycle_rule.condition.age": float64(30),
109122
},
110123
},
111124
},
112125
{
113126
name: "multiple resources",
114127
hcl: multipleResourcesHCL,
115-
exp: map[string]map[string]struct{}{
128+
exp: map[string]map[string]any{
116129
"google_project_service.project": {
117-
"service": {},
130+
"service": "iam.googleapis.com",
118131
},
119132
"google_storage_bucket.bucket": {
120-
"name": {},
133+
"name": "my-bucket",
121134
},
122135
},
123136
},
124137
{
125138
name: "resource with a list of nested objects",
126139
hcl: listOfNestedObjectsHCL,
127-
exp: map[string]map[string]struct{}{
140+
exp: map[string]map[string]any{
128141
"google_compute_firewall.default": {
129-
"allow.0.ports": {}, // "ports" appears in first element due to sorting
130-
"allow.0.protocol": {},
131-
"allow.1.protocol": {},
132-
"name": {},
133-
"network": {},
134-
"source_tags": {},
142+
"allow.0.ports": "80,8080,1000-2000", // "ports" appears in first element due to sorting
143+
"allow.0.protocol": "tcp",
144+
"allow.1.protocol": "icmp",
145+
"name": "test-firewall",
146+
"network": "google_compute_network.default.name",
147+
"source_tags": "web",
135148
},
136149
},
137150
},
138151
{
139152
name: "resource with a list of multi-level nested objects",
140153
hcl: listOfMultiLevelNestedObjectsHCL,
141-
exp: map[string]map[string]struct{}{
154+
exp: map[string]map[string]any{
142155
"google_compute_firewall.default": {
143-
"allow.0.a_second_level.0.a": {},
144-
"allow.0.a_second_level.1.b": {},
145-
"allow.0.protocol": {},
146-
"allow.1.ports": {},
147-
"allow.1.protocol": {},
148-
"name": {},
149-
"network": {},
150-
"source_tags": {},
156+
"allow.0.a_second_level.0.a": false,
157+
"allow.0.a_second_level.1.b": true,
158+
"allow.0.protocol": "icmp",
159+
"allow.1.ports": "80,8080,1000-2000",
160+
"allow.1.protocol": "tcp",
161+
"name": "test-firewall",
162+
"network": "google_compute_network.default.name",
163+
"source_tags": "web",
164+
},
165+
},
166+
},
167+
{
168+
name: "resource with map",
169+
hcl: mapHCL,
170+
exp: map[string]map[string]any{
171+
"google_bigquery_dataset.dataset": {
172+
"dataset_id": "datasetjlgukul2im",
173+
"description": "This is a test description",
174+
"friendly_name": "test",
175+
"location": "EU",
176+
"project": "ci-test-project-nightly-beta",
177+
"resource_tags": map[string]any{},
151178
},
152179
},
153180
},

‎mmv1/third_party/tgc_next/test/setup.go‎

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ type TgcMetadataPayload struct {
4242
}
4343

4444
type ResourceTestData struct {
45-
ParsedRawConfig map[string]struct{} `json:"parsed_raw_config"`
45+
ParsedRawConfig map[string]any `json:"parsed_raw_config"`
4646
ResourceMetadata `json:"resource_metadata"`
4747
}
4848

@@ -53,9 +53,9 @@ type StepTestData struct {
5353
}
5454

5555
type Resource struct {
56-
Type string `json:"type"`
57-
Name string `json:"name"`
58-
Attributes map[string]struct{} `json:"attributes"`
56+
Type string `json:"type"`
57+
Name string `json:"name"`
58+
Attributes map[string]any `json:"attributes"`
5959
}
6060

6161
const (
@@ -205,7 +205,11 @@ func prepareTestData(testName string, stepNumber int, retries int) (*StepTestDat
205205
}
206206

207207
if len(rawResourceConfigs) == 0 {
208-
return nil, fmt.Errorf("Test %s fails: raw config is unavailable", testName)
208+
return nil, fmt.Errorf("test %s fails: raw config is unavailable", testName)
209+
}
210+
211+
if os.Getenv("WRITE_FILES") != "" {
212+
writeJSONFile(fmt.Sprintf("%s_attrs", testName), rawResourceConfigs)
209213
}
210214

211215
rawConfigMap := convertToConfigMap(rawResourceConfigs)
@@ -255,8 +259,8 @@ func parseResourceConfigs(filePath string) ([]Resource, error) {
255259
}
256260

257261
// Converts the slice to map with resource address as the key
258-
func convertToConfigMap(resources []Resource) map[string]map[string]struct{} {
259-
configMap := make(map[string]map[string]struct{}, 0)
262+
func convertToConfigMap(resources []Resource) map[string]map[string]any {
263+
configMap := make(map[string]map[string]any, 0)
260264

261265
for _, r := range resources {
262266
addr := fmt.Sprintf("%s.%s", r.Type, r.Name)

0 commit comments

Comments
 (0)