Config Generation: Even more references (#1595)

This PR builds upon #1585 and finds refs also from computed attributes
An example is the `org-prefs` test added. The org ID is a computed attribute but it is being figured out as a reference

Author: julienduchesne

pkg/generate/cloud.go

@@ -72,14 +72,17 @@ func generateCloudResources(ctx context.Context, cfg *Config) ([]stack, error) {
 		return nil, err
 	}
 
-	log.Println("Post-processing for cloud")
-	if err := stripDefaults(filepath.Join(cfg.OutputDir, "cloud-resources.tf"), nil); err != nil {
+	postprocessor := &postprocessor{}
+	if postprocessor.plannedState, err = getPlannedState(ctx, cfg); err != nil {
 		return nil, err
 	}
-	if err := wrapJSONFieldsInFunction(filepath.Join(cfg.OutputDir, "cloud-resources.tf")); err != nil {
+	if err := postprocessor.stripDefaults(filepath.Join(cfg.OutputDir, "cloud-resources.tf"), nil); err != nil {
 		return nil, err
 	}
-	if err := replaceReferences(filepath.Join(cfg.OutputDir, "cloud-resources.tf"), nil); err != nil {
+	if err := postprocessor.wrapJSONFieldsInFunction(filepath.Join(cfg.OutputDir, "cloud-resources.tf")); err != nil {
+		return nil, err
+	}
+	if err := postprocessor.replaceReferences(filepath.Join(cfg.OutputDir, "cloud-resources.tf"), nil); err != nil {
 		return nil, err
 	}
 

pkg/generate/generate.go

@@ -138,6 +138,7 @@ func generateImportBlocks(ctx context.Context, client *common.Client, listerData
 				}
 				return
 			}
+			sort.Strings(ids)
 
 			// Write blocks like these
 			// import {

pkg/generate/grafana.go

@@ -76,16 +76,20 @@ func generateGrafanaResources(ctx context.Context, cfg *Config, stack stack, gen
 		stripDefaultsExtraFields["org_id"] = `"1"` // Remove org_id if it's the default
 	}
 
-	if err := stripDefaults(generatedFilename("resources.tf"), stripDefaultsExtraFields); err != nil {
+	postprocessor := &postprocessor{}
+	if postprocessor.plannedState, err = getPlannedState(ctx, cfg); err != nil {
 		return err
 	}
-	if err := abstractDashboards(generatedFilename("resources.tf")); err != nil {
+	if err := postprocessor.stripDefaults(generatedFilename("resources.tf"), stripDefaultsExtraFields); err != nil {
 		return err
 	}
-	if err := wrapJSONFieldsInFunction(generatedFilename("resources.tf")); err != nil {
+	if err := postprocessor.abstractDashboards(generatedFilename("resources.tf")); err != nil {
 		return err
 	}
-	if err := replaceReferences(generatedFilename("resources.tf"), []string{
+	if err := postprocessor.wrapJSONFieldsInFunction(generatedFilename("resources.tf")); err != nil {
+		return err
+	}
+	if err := postprocessor.replaceReferences(generatedFilename("resources.tf"), []string{
 		"*.org_id=grafana_organization.id",
 	}); err != nil {
 		return err

pkg/generate/postprocessing.go

@@ -13,6 +13,7 @@ import (
 	"github.com/hashicorp/hcl/v2"
 	"github.com/hashicorp/hcl/v2/hclsyntax"
 	"github.com/hashicorp/hcl/v2/hclwrite"
+	tfjson "github.com/hashicorp/terraform-json"
 	"github.com/zclconf/go-cty/cty"
 )
 
@@ -138,9 +139,12 @@ var knownReferences = []string{
 	"grafana_team_preferences.team_id=grafana_team.id",
 }
 
-// TODO: Also find references from the state (computed fields, like ID)
-func replaceReferences(fpath string, extraKnownReferences []string) error {
-	file, err := readHCLFile(fpath)
+type postprocessor struct {
+	plannedState *tfjson.Plan
+}
+
+func (p *postprocessor) replaceReferences(fpath string, extraKnownReferences []string) error {
+	file, err := p.readHCLFile(fpath)
 	if err != nil {
 		return err
 	}
@@ -149,17 +153,23 @@ func replaceReferences(fpath string, extraKnownReferences []string) error {
 
 	knownReferences := knownReferences
 	knownReferences = append(knownReferences, extraKnownReferences...)
-	// Find all resources. This map will be used to search for references
-	resourcesBlocks := map[string]*hclwrite.Block{}
+
+	plannedResources := p.plannedState.PlannedValues.RootModule.Resources
+
 	for _, block := range file.Body().Blocks() {
-		if block.Type() == "resource" {
-			resourcesBlocks[block.Labels()[0]+"."+block.Labels()[1]] = block
+		var blockResource *tfjson.StateResource
+		for _, plannedResource := range plannedResources {
+			if plannedResource.Type == block.Labels()[0] && plannedResource.Name == block.Labels()[1] {
+				blockResource = plannedResource
+				break
+			}
+		}
+		if blockResource == nil {
+			return fmt.Errorf("resource %s.%s not found in planned state", block.Labels()[0], block.Labels()[1])
 		}
-	}
 
-	for _, block := range file.Body().Blocks() {
-		for attrName, attr := range block.Body().Attributes() {
-			attrValue := string(attr.Expr().BuildTokens(nil).Bytes())
+		for attrName := range block.Body().Attributes() {
+			attrValue := blockResource.AttributeValues[attrName]
 			attrReplaced := false
 
 			// Check the field name. If it has a possible reference, we have to search for it in the resources
@@ -178,20 +188,19 @@ func replaceReferences(fpath string, extraKnownReferences []string) error {
 				refToResource := strings.Split(refTo, ".")[0]
 				refToAttr := strings.Split(refTo, ".")[1]
 
-				for possibleResourceRefName, possibleResourceRef := range resourcesBlocks {
-					if strings.HasPrefix(possibleResourceRefName, refToResource+".") {
-						valueFromRef := ""
-						if possibleResourceRef.Body().GetAttribute(refToAttr) != nil {
-							valueFromRef = string(possibleResourceRef.Body().GetAttribute(refToAttr).Expr().BuildTokens(nil).Bytes())
-						}
-						// If the value from the first block matches the value from the second block, we have a reference
-						if attrValue == valueFromRef {
-							// Replace the value with the reference
-							block.Body().SetAttributeTraversal(attrName, traversal(possibleResourceRefName, refToAttr))
-							hasChanges = true
-							attrReplaced = true
-							break
-						}
+				for _, plannedResource := range plannedResources {
+					if plannedResource.Type != refToResource {
+						continue
+					}
+
+					valueFromRef := plannedResource.AttributeValues[refToAttr]
+					// If the value from the first block matches the value from the second block, we have a reference
+					if attrValue == valueFromRef {
+						// Replace the value with the reference
+						block.Body().SetAttributeTraversal(attrName, traversal(plannedResource.Type, plannedResource.Name, refToAttr))
+						hasChanges = true
+						attrReplaced = true
+						break
 					}
 				}
 			}
@@ -206,15 +215,15 @@ func replaceReferences(fpath string, extraKnownReferences []string) error {
 	return nil
 }
 
-func stripDefaults(fpath string, extraFieldsToRemove map[string]any) error {
-	file, err := readHCLFile(fpath)
+func (p *postprocessor) stripDefaults(fpath string, extraFieldsToRemove map[string]any) error {
+	file, err := p.readHCLFile(fpath)
 	if err != nil {
 		return err
 	}
 
 	hasChanges := false
 	for _, block := range file.Body().Blocks() {
-		if s := stripDefaultsFromBlock(block, extraFieldsToRemove); s {
+		if s := p.stripDefaultsFromBlock(block, extraFieldsToRemove); s {
 			hasChanges = true
 		}
 	}
@@ -225,8 +234,8 @@ func stripDefaults(fpath string, extraFieldsToRemove map[string]any) error {
 	return nil
 }
 
-func wrapJSONFieldsInFunction(fpath string) error {
-	file, err := readHCLFile(fpath)
+func (p *postprocessor) wrapJSONFieldsInFunction(fpath string) error {
+	file, err := p.readHCLFile(fpath)
 	if err != nil {
 		return err
 	}
@@ -235,7 +244,7 @@ func wrapJSONFieldsInFunction(fpath string) error {
 	// Find json attributes and use jsonencode
 	for _, block := range file.Body().Blocks() {
 		for key, attr := range block.Body().Attributes() {
-			asMap, err := attributeToMap(attr)
+			asMap, err := p.attributeToMap(attr)
 			if err != nil || asMap == nil {
 				continue
 			}
@@ -252,11 +261,11 @@ func wrapJSONFieldsInFunction(fpath string) error {
 	return nil
 }
 
-func abstractDashboards(fpath string) error {
+func (p *postprocessor) abstractDashboards(fpath string) error {
 	fDir := filepath.Dir(fpath)
 	outPath := filepath.Join(fDir, "files")
 
-	file, err := readHCLFile(fpath)
+	file, err := p.readHCLFile(fpath)
 	if err != nil {
 		return err
 	}
@@ -269,7 +278,7 @@ func abstractDashboards(fpath string) error {
 			continue
 		}
 
-		dashboard, err := attributeToJSON(block.Body().GetAttribute("config_json"))
+		dashboard, err := p.attributeToJSON(block.Body().GetAttribute("config_json"))
 		if err != nil {
 			return err
 		}
@@ -316,7 +325,7 @@ func abstractDashboards(fpath string) error {
 	return nil
 }
 
-func attributeToMap(attr *hclwrite.Attribute) (map[string]interface{}, error) {
+func (p *postprocessor) attributeToMap(attr *hclwrite.Attribute) (map[string]interface{}, error) {
 	var err error
 
 	// Convert jsonencode to raw json
@@ -345,8 +354,8 @@ func attributeToMap(attr *hclwrite.Attribute) (map[string]interface{}, error) {
 	return dashboardMap, nil
 }
 
-func attributeToJSON(attr *hclwrite.Attribute) ([]byte, error) {
-	jsonMap, err := attributeToMap(attr)
+func (p *postprocessor) attributeToJSON(attr *hclwrite.Attribute) ([]byte, error) {
+	jsonMap, err := p.attributeToMap(attr)
 	if err != nil || jsonMap == nil {
 		return nil, err
 	}
@@ -359,7 +368,7 @@ func attributeToJSON(attr *hclwrite.Attribute) ([]byte, error) {
 	return jsonMarshalled, nil
 }
 
-func readHCLFile(fpath string) (*hclwrite.File, error) {
+func (p *postprocessor) readHCLFile(fpath string) (*hclwrite.File, error) {
 	src, err := os.ReadFile(fpath)
 	if err != nil {
 		return nil, err
@@ -373,10 +382,10 @@ func readHCLFile(fpath string) (*hclwrite.File, error) {
 	return file, nil
 }
 
-func stripDefaultsFromBlock(block *hclwrite.Block, extraFieldsToRemove map[string]any) bool {
+func (p *postprocessor) stripDefaultsFromBlock(block *hclwrite.Block, extraFieldsToRemove map[string]any) bool {
 	hasChanges := false
 	for _, innblock := range block.Body().Blocks() {
-		if s := stripDefaultsFromBlock(innblock, extraFieldsToRemove); s {
+		if s := p.stripDefaultsFromBlock(innblock, extraFieldsToRemove); s {
 			hasChanges = true
 		}
 		if len(innblock.Body().Attributes()) == 0 && len(innblock.Body().Blocks()) == 0 {
@@ -405,7 +414,7 @@ func stripDefaultsFromBlock(block *hclwrite.Block, extraFieldsToRemove map[strin
 			if name == key {
 				toRemove := false
 				fieldValue := strings.TrimSpace(string(attribute.Expr().BuildTokens(nil).Bytes()))
-				fieldValue, err := extractJSONEncode(fieldValue)
+				fieldValue, err := p.extractJSONEncode(fieldValue)
 				if err != nil {
 					continue
 				}
@@ -425,6 +434,16 @@ func stripDefaultsFromBlock(block *hclwrite.Block, extraFieldsToRemove map[strin
 	}
 	return hasChanges
 }
+func (p *postprocessor) extractJSONEncode(value string) (string, error) {
+	if !strings.HasPrefix(value, "jsonencode(") {
+		return "", nil
+	}
+	value = strings.TrimPrefix(value, "jsonencode(")
+	value = strings.TrimSuffix(value, ")")
+
+	b, err := json.MarshalIndent(value, "", "  ")
+	return string(b), err
+}
 
 // BELOW IS FROM https://github.com/hashicorp/terraform/blob/main/internal/configs/hcl2shim/values.go
 
@@ -472,14 +491,3 @@ func HCL2ValueFromConfigValue(v interface{}) cty.Value {
 		panic(fmt.Errorf("can't convert %#v to cty.Value", v))
 	}
 }
-
-func extractJSONEncode(value string) (string, error) {
-	if !strings.HasPrefix(value, "jsonencode(") {
-		return "", nil
-	}
-	value = strings.TrimPrefix(value, "jsonencode(")
-	value = strings.TrimSuffix(value, ")")
-
-	b, err := json.MarshalIndent(value, "", "  ")
-	return string(b), err
-}

pkg/generate/terraform_state.go

@@ -3,7 +3,10 @@ package generate
 import (
 	"context"
 	"fmt"
+	"os"
+	"path/filepath"
 
+	"github.com/hashicorp/terraform-exec/tfexec"
 	tfjson "github.com/hashicorp/terraform-json"
 )
 
@@ -14,3 +17,18 @@ func getState(ctx context.Context, cfg *Config) (*tfjson.State, error) {
 	}
 	return state, nil
 }
+
+func getPlannedState(ctx context.Context, cfg *Config) (*tfjson.Plan, error) {
+	tempWorkingDir, err := os.MkdirTemp("", "terraform-generate")
+	if err != nil {
+		return nil, fmt.Errorf("failed to create temporary working directory: %w", err)
+	}
+	defer os.RemoveAll(tempWorkingDir)
+
+	planFile := filepath.Join(tempWorkingDir, "plan.tfplan")
+	if _, err := cfg.Terraform.Plan(ctx, tfexec.Out(planFile)); err != nil {
+		return nil, fmt.Errorf("failed to read terraform plan: %w", err)
+	}
+
+	return cfg.Terraform.ShowPlanFile(ctx, planFile)
+}

pkg/generate/testdata/generate/alerting-in-org/imports.tf

@@ -13,23 +13,18 @@ import {
   id = "2:My Mute Timing"
 }
 
-import {
-  to = grafana_notification_policy._2_policy
-  id = "2:policy"
-}
-
 import {
   to = grafana_notification_policy._1_policy
   id = "1:policy"
 }
 
 import {
-  to = grafana_organization._2
-  id = "2"
+  to = grafana_notification_policy._2_policy
+  id = "2:policy"
 }
 
 import {
-  to = grafana_organization_preferences._2
+  to = grafana_organization._2
   id = "2"
 }
 
@@ -38,6 +33,11 @@ import {
   id = "1"
 }
 
+import {
+  to = grafana_organization_preferences._2
+  id = "2"
+}
+
 import {
   to = grafana_rule_group._2_alert-rule-folder_My_Rule_Group
   id = "2:alert-rule-folder:My Rule Group"