Merge pull request #662 from ericzbeard/awscli-modules

This PR makes significant changes to the functionality for modules, making Rain compatible with a draft PR that is in progress for the AWS CLI `cloudformation package` command.

Author: ericzbeard

README.md

@@ -30,20 +30,21 @@ type PackageAlias struct {
 
 // Template represents a CloudFormation template. The Template type
 // is minimal for now but will likely grow new features as needed by rain.
+//
+// Node is the only member that is guaranteed to exist after
+// parsing a template.
 type Template struct {
-	Node *yaml.Node
-
-	Constants map[string]*yaml.Node
-	Packages  map[string]*PackageAlias
+	FileName       string
+	Name           string
+	Node           *yaml.Node
+	Constants      map[string]*yaml.Node
+	Packages       map[string]*PackageAlias
+	ModuleMapNames map[string][]string
+	ModuleMaps     map[string]*ModuleConfig
+	ModuleOutputs  map[string]*yaml.Node
+	ModuleResolved []*yaml.Node
 }
 
-// TODO - We really need a convenient Template data structure
-// that lets us easily access elements.
-// t.Resources["MyResource"].Properties["MyProp"]
-//
-// Add a Model attribute to the struct and an Init function to populate it.
-// t.Model.Resources
-
 // Map returns the template as a map[string]interface{}
 func (t Template) Map() map[string]interface{} {
 	var out map[string]interface{}
@@ -57,7 +58,7 @@ func (t Template) Map() map[string]interface{} {
 }
 
 // AppendStateMap appends a "State" section to the template
-func AppendStateMap(state Template) *yaml.Node {
+func AppendStateMap(state *Template) *yaml.Node {
 	state.Node.Content[0].Content = append(state.Node.Content[0].Content,
 		&yaml.Node{Kind: yaml.ScalarNode, Value: "State"})
 	stateMap := &yaml.Node{Kind: yaml.MappingNode, Content: make([]*yaml.Node, 0)}
@@ -81,6 +82,18 @@ const (
 	Outputs                  Section = "Outputs"
 	State                    Section = "State"
 	Rain                     Section = "Rain"
+	Modules                  Section = "Modules"
+	Packages                 Section = "Packages"
+	Constants                Section = "Constants"
+)
+
+type Intrinsic string
+
+const (
+	Sub    Intrinsic = "Fn::Sub"
+	GetAtt Intrinsic = "Fn::GetAtt"
+	Ref    Intrinsic = "Ref"
+	If     Intrinsic = "Fn::If"
 )
 
 // GetResource returns the yaml node for a resource by logical id
@@ -148,9 +161,19 @@ func (t Template) GetSection(section Section) (*yaml.Node, error) {
 	return s, nil
 }
 
+// HasSection returns true if the template has the section
+func (t Template) HasSection(section Section) bool {
+	if t.Node == nil {
+		return false
+	}
+	m := t.Node.Content[0]
+	_, s, _ := s11n.GetMapValue(m, string(section))
+	return s != nil
+}
+
 // RemoveSection removes a section node from the template
 func (t Template) RemoveSection(section Section) error {
-	return node.RemoveFromMap(t.Node.Content[0], string(Rain))
+	return node.RemoveFromMap(t.Node.Content[0], string(section))
 }
 
 // GetTypes returns all unique type names for resources in the template
@@ -224,3 +247,38 @@ func (t Template) RemoveEmptySections() {
 		node.RemoveFromMap(m, name)
 	}
 }
+
+// AddMappedModule adds a reference to a module that was mapped to a CSV of keys,
+// which duplicates the module in the template. We store a reference here so
+// that we can resolve references like Content[0].Arn, which points to the first
+// mapped instance of a Module called Content, with an Output called Arn.
+func (t *Template) AddMappedModule(copiedConfig *ModuleConfig) {
+	if t.ModuleMaps == nil {
+		t.ModuleMaps = make(map[string]*ModuleConfig)
+	}
+	t.ModuleMaps[copiedConfig.Name] = copiedConfig
+	if t.ModuleMapNames == nil {
+		t.ModuleMapNames = make(map[string][]string)
+	}
+	originalName := copiedConfig.OriginalName
+	var mappedModules []string
+	var ok bool
+	if mappedModules, ok = t.ModuleMapNames[originalName]; !ok {
+		mappedModules = make([]string, 0)
+	}
+	if !slices.Contains(mappedModules, copiedConfig.Name) {
+		mappedModules = append(mappedModules, copiedConfig.Name)
+	}
+	t.ModuleMapNames[originalName] = mappedModules
+}
+
+func (t *Template) AddResolvedModuleNode(n *yaml.Node) {
+	if t.ModuleResolved == nil {
+		t.ModuleResolved = make([]*yaml.Node, 0)
+	}
+	t.ModuleResolved = append(t.ModuleResolved, n)
+}
+
+func (t *Template) ModuleAlreadyResolved(n *yaml.Node) bool {
+	return slices.Contains(t.ModuleResolved, n)
+}

cft/cft.go

@@ -8,7 +8,7 @@ import (
 )
 
 // New returns a Diff that represents the difference between two templates
-func New(a, b cft.Template) Diff {
+func New(a, b *cft.Template) Diff {
 	return CompareMaps(a.Map(), b.Map())
 }
 

cft/diff/compare.go

@@ -30,7 +30,7 @@ func CheckMultilineBegin(s string) bool {
 }
 
 // String returns a string representation of the supplied cft.Template
-func String(t cft.Template, opt Options) string {
+func String(t *cft.Template, opt Options) string {
 	node := t.Node
 
 	buf := strings.Builder{}
@@ -122,6 +122,6 @@ func String(t cft.Template, opt Options) string {
 }
 
 // CftToYaml converts a template to a YAML string
-func CftToYaml(t cft.Template) string {
+func CftToYaml(t *cft.Template) string {
 	return String(t, Options{JSON: false, Unsorted: false})
 }

cft/format/format.go

@@ -457,7 +457,7 @@ func addSection(section cft.Section, n *yaml.Node, sb *strings.Builder, basic bo
 
 // CftToPkl serializes the template as pkl.
 // It assumes that the user is importing the cloudformation package
-func CftToPkl(t cft.Template, basic bool, pklPackageAlias string) (string, error) {
+func CftToPkl(t *cft.Template, basic bool, pklPackageAlias string) (string, error) {
 	if t.Node == nil || len(t.Node.Content) != 1 {
 		return "", errors.New("expected t.Node.Content[0]")
 	}

cft/format/pkl.go

@@ -12,7 +12,7 @@ import (
 	"gopkg.in/yaml.v3"
 )
 
-func UnCDK(t cft.Template) error {
+func UnCDK(t *cft.Template) error {
 
 	// Remove these nodes:
 	//
@@ -153,7 +153,7 @@ func joinSeqToString(seq *yaml.Node) string {
 	return retval
 }
 
-func joinToSub(t cft.Template) {
+func joinToSub(t *cft.Template) {
 	vf := func(n *visitor.Visitor) {
 		yamlNode := n.GetYamlNode()
 		if yamlNode.Kind == yaml.MappingNode {
@@ -173,7 +173,7 @@ func joinToSub(t cft.Template) {
 
 }
 
-func replaceNames(t cft.Template, oldName, newName string) {
+func replaceNames(t *cft.Template, oldName, newName string) {
 	vf := func(n *visitor.Visitor) {
 		yamlNode := n.GetYamlNode()
 		if yamlNode.Kind == yaml.ScalarNode {
@@ -187,7 +187,7 @@ func replaceNames(t cft.Template, oldName, newName string) {
 }
 
 // getCommonTemplatePrefix attempts to find a common string that begins all resource names.
-func getCommonResourcePrefix(t cft.Template) string {
+func getCommonResourcePrefix(t *cft.Template) string {
 	resources, err := t.GetSection(cft.Resources)
 	if err != nil {
 		return ""

cft/format/uncdk.go

@@ -43,7 +43,7 @@ func Empty() Graph {
 // New returns a Graph representing the connections
 // between elements in the provided template.
 // The type of each item in the graph is Node
-func New(t cft.Template) Graph {
+func New(t *cft.Template) Graph {
 	// Map out parameter and resource names so we know which is which
 	entities := make(map[string]string)
 	for typeName, entity := range t.Map() {

cft/graph/graph.go

@@ -32,7 +32,7 @@ Outputs:
     Value: !GetAtt Bucket.Arn
 `
 
-var template cft.Template
+var template *cft.Template
 
 var g graph.Graph
 

cft/graph/graph_test.go

@@ -0,0 +1,119 @@
+package cft
+
+import (
+	"errors"
+
+	"github.com/aws-cloudformation/rain/internal/node"
+	"github.com/aws-cloudformation/rain/internal/s11n"
+	"gopkg.in/yaml.v3"
+)
+
+// ModuleConfig is the configuration of the module in the parent template.
+type ModuleConfig struct {
+
+	// Name is the name of the module, which is used as a logical id prefix
+	Name string
+
+	// Source is the URI for the module, a local file or remote URL
+	Source string
+
+	// PropertiesNode is the yaml node for the Properties
+	PropertiesNode *yaml.Node
+
+	// OverridesNode is the yaml node for overrides
+	OverridesNode *yaml.Node
+
+	// Node is the yaml node for the Module config mapping node
+	Node *yaml.Node
+
+	// Map is the node for mapping (duplicating) the module based on a CSV
+	Map *yaml.Node
+
+	// OriginalName is the Name of the module before it got Mapped (duplicated)
+	OriginalName string
+
+	// If this module was duplicated because of the Map attribute, store the index
+	MapIndex int
+
+	// If this module was duplicated because of the Map attribute, store the key
+	MapKey string
+
+	// IsMapCopy will be true if this instance was a duplicate of a Mapped module
+	IsMapCopy bool
+
+	// The root directory of the template that configures this module
+	ParentRootDir string
+}
+
+func (c *ModuleConfig) Properties() map[string]any {
+	return node.DecodeMap(c.PropertiesNode)
+}
+
+func (c *ModuleConfig) Overrides() map[string]any {
+	return node.DecodeMap(c.OverridesNode)
+}
+
+// ResourceOverridesNode returns the Overrides node for the given resource if it exists
+func (c *ModuleConfig) ResourceOverridesNode(name string) *yaml.Node {
+	if c.OverridesNode == nil {
+		return nil
+	}
+	_, n, _ := s11n.GetMapValue(c.OverridesNode, name)
+	return n
+}
+
+const (
+	Source     string = "Source"
+	Properties string = "Properties"
+	Overrides  string = "Overrides"
+	Map        string = "Map"
+)
+
+// parseModuleConfig parses a single module configuration
+// from the Modules section in the template
+func (t *Template) ParseModuleConfig(name string, n *yaml.Node) (*ModuleConfig, error) {
+	if n.Kind != yaml.MappingNode {
+		return nil, errors.New("not a mapping node")
+	}
+	m := &ModuleConfig{}
+	m.Name = name
+	m.Node = n
+
+	content := n.Content
+	for i := 0; i < len(content); i += 2 {
+		attr := content[i].Value
+		val := content[i+1]
+		switch attr {
+		case Source:
+			m.Source = val.Value
+		case Properties:
+			m.PropertiesNode = val
+		case Overrides:
+			m.OverridesNode = val
+		case Map:
+			m.Map = val
+		}
+	}
+
+	//err := t.ValidateModuleConfig(m)
+	//if err != nil {
+	//	return nil, err
+	//}
+
+	return m, nil
+}
+
+// ValidateModuleConfig makes sure the configuration does not
+// break any rules, such as not having a Property with the
+// same name as a Parameter.
+//func (t *Template) ValidateModuleConfig(moduleConfig *ModuleConfig) error {
+//	props := moduleConfig.Properties()
+//	for key := range props {
+//		_, err := t.GetParameter(key)
+//		if err == nil {
+//			return fmt.Errorf("module %s in %s has Property %s with the same name as a template Parameter",
+//				moduleConfig.Name, moduleConfig.ParentRootDir, key)
+//		}
+//	}
+//	return nil
+//}